diff options
1065 files changed, 12351 insertions, 11856 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 5463c4215a0..c995007663d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,8 @@ vespa_use_default_cmake_prefix_path() SET(CMAKE_FIND_PACKAGE_SORT_ORDER NATURAL) SET(CMAKE_FIND_PACKAGE_SORT_DIRECTION DEC) +find_package(Threads REQUIRED) + find_package(LLVM REQUIRED CONFIG) message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") @@ -66,7 +68,6 @@ include_directories(BEFORE ${CMAKE_BINARY_DIR}/configdefinitions/src) add_subdirectory(airlift-zstd) add_subdirectory(ann_benchmark) add_subdirectory(application-model) -add_subdirectory(athenz-identity-provider-service) add_subdirectory(client) add_subdirectory(cloud-tenant-cd) add_subdirectory(clustercontroller-apps) @@ -99,7 +100,6 @@ add_subdirectory(docprocs) add_subdirectory(document) add_subdirectory(documentapi) add_subdirectory(eval) -add_subdirectory(fastos) add_subdirectory(fbench) add_subdirectory(fileacquirer) add_subdirectory(filedistribution) diff --git a/README-cmake.md b/README-cmake.md index 214eaaa94a9..393e068d089 100644 --- a/README-cmake.md +++ b/README-cmake.md @@ -106,7 +106,6 @@ The module definition is used to specify common dependencies for every target de vespa_define_module( DEPENDS - fastos vespalib bjarnelib diff --git a/application/.gitignore b/application/.gitignore index f5d341f2123..bc385f92fa6 100644 --- a/application/.gitignore +++ b/application/.gitignore @@ -2,3 +2,4 @@ pom.xml.build /logs .preprocessed +/src/test/app-packages/*/models.generated/ diff --git a/application/abi-spec.json b/application/abi-spec.json index 7de80374b5f..c95039b4c1f 100644 --- a/application/abi-spec.json +++ b/application/abi-spec.json @@ -38,7 +38,8 @@ "public com.yahoo.application.Application$Builder rankExpression(java.lang.String, java.lang.String)", "public com.yahoo.application.Application$Builder queryProfile(java.lang.String, java.lang.String)", "public com.yahoo.application.Application$Builder queryProfileType(java.lang.String, java.lang.String)", - "public com.yahoo.application.Application$Builder networking(com.yahoo.application.Networking)" + "public com.yahoo.application.Application$Builder networking(com.yahoo.application.Networking)", + "public com.yahoo.application.Application build()" ], "fields" : [ ] }, diff --git a/application/pom.xml b/application/pom.xml index 2193f0fe2e3..236bcb6d81a 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -182,6 +182,12 @@ <artifactId>junit-jupiter-engine</artifactId> <scope>test</scope> </dependency> + <dependency> + <!-- Required for ContainerModelEvaluationTest --> + <groupId>com.microsoft.onnxruntime</groupId> + <artifactId>onnxruntime</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> diff --git a/application/src/main/java/com/yahoo/application/Application.java b/application/src/main/java/com/yahoo/application/Application.java index b4857d18459..d9234584630 100644 --- a/application/src/main/java/com/yahoo/application/Application.java +++ b/application/src/main/java/com/yahoo/application/Application.java @@ -345,7 +345,7 @@ public final class Application implements AutoCloseable { } // generate the services xml and load the container - private Application build() throws Exception { + public Application build() throws Exception { Application app = null; Exception exception = null; diff --git a/application/src/test/java/com/yahoo/application/ApplicationTest.java b/application/src/test/java/com/yahoo/application/ApplicationTest.java index 5b4a68756f0..6b394cdebd9 100644 --- a/application/src/test/java/com/yahoo/application/ApplicationTest.java +++ b/application/src/test/java/com/yahoo/application/ApplicationTest.java @@ -2,6 +2,7 @@ package com.yahoo.application; import com.yahoo.application.container.MockServer; +import com.yahoo.application.container.components.ComponentWithMetrics; import com.yahoo.application.container.docprocs.MockDocproc; import com.yahoo.application.container.handler.Request; import com.yahoo.application.container.handler.Response; @@ -15,6 +16,8 @@ import com.yahoo.docproc.Processing; import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentType; import com.yahoo.jdisc.handler.RequestHandler; +import com.yahoo.metrics.simple.Bucket; +import com.yahoo.metrics.simple.jdisc.SimpleMetricConsumer; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.handler.SearchHandler; @@ -263,6 +266,25 @@ public class ApplicationTest { } @Test + void application_generation_metric() throws Exception { + try (ApplicationFacade app = new ApplicationFacade(Application.fromBuilder(new Application.Builder().container("default", new Application.Builder.Container() + .component(ComponentWithMetrics.class))))) { + var component = (ComponentWithMetrics)app.getComponentById(ComponentWithMetrics.class.getName()); + assertNotNull(component); + var metrics = (SimpleMetricConsumer)component.metrics().newInstance(); // not actually a new instance + assertNotNull(metrics); + int maxWaitMs = 10000; + Bucket snapshot = null; + while (maxWaitMs-- > 0 && ( snapshot = metrics.receiver().getSnapshot() ) == null) { + Thread.sleep(1); + } + assertNotNull(snapshot); + assertEquals(1, snapshot.getValuesForMetric("application_generation").size()); + assertEquals(0, snapshot.getValuesForMetric("application_generation").iterator().next().getValue().getLast()); + } + } + + @Test void component_with_config() throws Exception { MockApplicationConfig config = new MockApplicationConfig(new MockApplicationConfig.Builder().mystruct(new MockApplicationConfig.Mystruct.Builder().id("foo").value("bar"))); try (ApplicationFacade app = new ApplicationFacade(Application.fromBuilder(new Application.Builder().container("default", new Application.Builder.Container() diff --git a/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java b/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java index f838d7a5481..c9ff51b0d84 100644 --- a/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java +++ b/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.application.container; -import ai.vespa.modelintegration.evaluator.OnnxEvaluator; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import com.yahoo.application.Application; import com.yahoo.application.Networking; import com.yahoo.application.container.handler.Request; @@ -40,7 +40,7 @@ public class ContainerModelEvaluationTest { @Test void testCreateApplicationInstanceWithModelEvaluation() { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); try (Application application = Application.fromApplicationPackage(new File("src/test/app-packages/model-evaluation"), Networking.disable)) { @@ -54,17 +54,17 @@ public class ContainerModelEvaluationTest { } { - String expected = "{\"cells\":[{\"address\":{},\"value\":2.496898}]}"; + String expected = "{\"type\":\"tensor()\",\"cells\":[{\"address\":{},\"value\":2.496898}]}"; assertResponse("http://localhost/model-evaluation/v1/xgboost_xgboost_2_2/eval?format.tensors=long", expected, jdisc); } { - String expected = "{\"cells\":[{\"address\":{},\"value\":1.9130086820218188}]}"; + String expected = "{\"type\":\"tensor()\",\"cells\":[{\"address\":{},\"value\":1.9130086820218188}]}"; assertResponse("http://localhost/model-evaluation/v1/lightgbm_regression/eval?format.tensors=long", expected, jdisc); } { - String expected = "{\"cells\":[{\"address\":{\"d0\":\"0\"},\"value\":0.3006095290184021},{\"address\":{\"d0\":\"1\"},\"value\":0.33222490549087524},{\"address\":{\"d0\":\"2\"},\"value\":0.3671652674674988}]}"; + String expected = "{\"type\":\"tensor<float>(d0[3])\",\"cells\":[{\"address\":{\"d0\":\"0\"},\"value\":0.3006095290184021},{\"address\":{\"d0\":\"1\"},\"value\":0.33222490549087524},{\"address\":{\"d0\":\"2\"},\"value\":0.36716532707214355}]}"; assertResponse("http://localhost/model-evaluation/v1/onnx_softmax_func/output/eval?format.tensors=long&input=" + inputTensor(), expected, jdisc); assertResponse("http://localhost/model-evaluation/v1/onnx_softmax_func/default.output/eval?format.tensors=long&input=" + inputTensor(), expected, jdisc); assertResponse("http://localhost/model-evaluation/v1/onnx_softmax_func/default/output/eval?format.tensors=long&input=" + inputTensor(), expected, jdisc); @@ -75,7 +75,13 @@ public class ContainerModelEvaluationTest { private void assertResponse(String url, String expectedResponse, JDisc jdisc) { try { Response response = jdisc.handleRequest(new Request(url)); - JsonTestHelper.assertJsonEquals(expectedResponse, response.getBodyAsString()); + + // Truncate JSON encoded numbers having more than 6 digits after the decimal point + String pattern = "([0-9]+\\.[0-9]{6})[0-9]*"; + String normalizedExpectedResponse = expectedResponse.replaceAll(pattern, "$1"); + String normalizedActualResponse = response.getBodyAsString().replaceAll(pattern, "$1"); + + JsonTestHelper.assertJsonEquals(normalizedExpectedResponse, normalizedActualResponse); assertEquals(200, response.getStatus()); } catch (CharacterCodingException e) { diff --git a/application/src/test/java/com/yahoo/application/container/components/ComponentWithMetrics.java b/application/src/test/java/com/yahoo/application/container/components/ComponentWithMetrics.java new file mode 100644 index 00000000000..45bb9bd846d --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/components/ComponentWithMetrics.java @@ -0,0 +1,27 @@ +package com.yahoo.application.container.components; + +import com.yahoo.metrics.simple.jdisc.JdiscMetricsFactory; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +/** + * @author bratseth + */ +public class ComponentWithMetrics extends Searcher { + + private final JdiscMetricsFactory metrics; + + public ComponentWithMetrics(JdiscMetricsFactory metrics) { + this.metrics = metrics; + } + + public JdiscMetricsFactory metrics() { return metrics; } + + @Override + public Result search(Query query, Execution execution) { + return execution.search(query); + } + +} diff --git a/athenz-identity-provider-service/CMakeLists.txt b/athenz-identity-provider-service/CMakeLists.txt deleted file mode 100644 index 75208c49bd3..00000000000 --- a/athenz-identity-provider-service/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_jar(athenz-identity-provider-service-jar-with-dependencies.jar) diff --git a/athenz-identity-provider-service/OWNERS b/athenz-identity-provider-service/OWNERS deleted file mode 100644 index 569bf1cc3a1..00000000000 --- a/athenz-identity-provider-service/OWNERS +++ /dev/null @@ -1 +0,0 @@ -bjorncs diff --git a/athenz-identity-provider-service/README.md b/athenz-identity-provider-service/README.md deleted file mode 100644 index b25eb009e1b..00000000000 --- a/athenz-identity-provider-service/README.md +++ /dev/null @@ -1,5 +0,0 @@ -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -# Athenz Identity Provider Service - -An [Athenz Copper Argos](https://github.com/yahoo/athenz/blob/master/docs/copper_argos.md) provider implementation for configserver. - diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml deleted file mode 100644 index f4daa43b8e3..00000000000 --- a/athenz-identity-provider-service/pom.xml +++ /dev/null @@ -1,186 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 - http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <artifactId>athenz-identity-provider-service</artifactId> - <packaging>container-plugin</packaging> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> - <version>8-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> - </parent> - <dependencies> - <!-- PROVIDED --> - <dependency> - <groupId>com.google.inject</groupId> - <artifactId>guice</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>component</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-apache-http-client-bundle</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-dev</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>jdisc_core</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>node-repository</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-provisioning</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-model-api</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespa-athenz</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>security-utils</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - - <!-- TEST --> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>zkfacade</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.junit.jupiter</groupId> - <artifactId>junit-jupiter-api</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.junit.jupiter</groupId> - <artifactId>junit-jupiter-engine</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>orchestrator</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>application</artifactId> - <version>${project.version}</version> - <scope>test</scope> - <exclusions> - <exclusion> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>testutil</artifactId> - <version>${project.version}</version> - <scope>test</scope> - <exclusions> - <exclusion> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - </exclusion> - <exclusion> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>flags</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-test</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - </dependencies> - - <build> - <plugins> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugin</artifactId> - <extensions>true</extensions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <!-- Illegal reflective access by guice. TODO: try to remove for guice >3.0 --> - <argLine> - --add-opens=java.base/java.lang=ALL-UNNAMED - </argLine> - </configuration> - </plugin> - </plugins> - </build> - -</project> diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CertificateExpiryMetricUpdater.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CertificateExpiryMetricUpdater.java deleted file mode 100644 index f3568caac04..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CertificateExpiryMetricUpdater.java +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.component.AbstractComponent; -import com.yahoo.jdisc.Metric; - -import java.time.Duration; -import java.time.Instant; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * @author freva - */ -public class CertificateExpiryMetricUpdater extends AbstractComponent { - - private static final Duration METRIC_REFRESH_PERIOD = Duration.ofMinutes(5); - private static final String ATHENZ_CONFIGSERVER_CERT_METRIC_NAME = "athenz-configserver-cert.expiry.seconds"; - - private final Logger logger = Logger.getLogger(CertificateExpiryMetricUpdater.class.getName()); - private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); - private final Metric metric; - private final ConfigserverSslContextFactoryProvider provider; - - @Inject - public CertificateExpiryMetricUpdater(Metric metric, - ConfigserverSslContextFactoryProvider provider) { - this.metric = metric; - this.provider = provider; - - scheduler.scheduleAtFixedRate(this::updateMetrics, - 30/*initial delay*/, - METRIC_REFRESH_PERIOD.getSeconds(), - TimeUnit.SECONDS); - } - - @Override - public void deconstruct() { - try { - scheduler.shutdownNow(); - scheduler.awaitTermination(30, TimeUnit.SECONDS); - } catch (InterruptedException e) { - throw new RuntimeException("Failed to shutdown certificate expiry metrics updater on time", e); - } - } - - private void updateMetrics() { - try { - Duration keyStoreExpiry = Duration.between(Instant.now(), provider.getCertificateNotAfter()); - metric.set(ATHENZ_CONFIGSERVER_CERT_METRIC_NAME, keyStoreExpiry.getSeconds(), null); - } catch (Exception e) { - logger.log(Level.WARNING, "Failed to update key store expiry metric: " + e.getMessage(), e); - } - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CkmsKeyProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CkmsKeyProvider.java deleted file mode 100644 index c659c454420..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CkmsKeyProvider.java +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.config.provision.Zone; -import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.security.KeyUtils; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.KeyProvider; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; - -import java.security.KeyPair; -import java.security.PrivateKey; -import java.security.PublicKey; -import java.util.HashMap; -import java.util.Map; - -/** - * @author mortent - * @author bjorncs - */ -@SuppressWarnings("unused") // Injected component -public class CkmsKeyProvider implements KeyProvider { - - private final SecretStore secretStore; - private final String secretName; - private final Map<Integer, KeyPair> secrets; - - @Inject - public CkmsKeyProvider(SecretStore secretStore, - Zone zone, - AthenzProviderServiceConfig config) { - this.secretStore = secretStore; - this.secretName = config.secretName(); - this.secrets = new HashMap<>(); - } - - @Override - public PrivateKey getPrivateKey(int version) { - return getKeyPair(version).getPrivate(); - } - - @Override - public PublicKey getPublicKey(int version) { - return getKeyPair(version).getPublic(); - } - - @Override - public KeyPair getKeyPair(int version) { - synchronized (secrets) { - KeyPair keyPair = secrets.get(version); - if (keyPair == null) { - keyPair = readKeyPair(version); - secrets.put(version, keyPair); - } - return keyPair; - } - } - - private KeyPair readKeyPair(int version) { - PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(secretStore.getSecret(secretName, version)); - PublicKey publicKey = KeyUtils.extractPublicKey(privateKey); - return new KeyPair(publicKey, privateKey); - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java deleted file mode 100644 index 61a4a0fe41f..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.jdisc.http.ssl.impl.TlsContextBasedProvider; -import com.yahoo.security.KeyStoreBuilder; -import com.yahoo.security.KeyStoreType; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.SslContextBuilder; -import com.yahoo.security.tls.DefaultTlsContext; -import com.yahoo.security.MutableX509KeyManager; -import com.yahoo.security.tls.PeerAuthentication; -import com.yahoo.security.tls.TlsContext; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; -import com.yahoo.vespa.athenz.client.zts.Identity; -import com.yahoo.vespa.athenz.client.zts.ZtsClient; -import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; -import com.yahoo.vespa.athenz.utils.SiaUtils; -import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; - -import javax.net.ssl.SSLContext; -import java.net.URI; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.security.KeyPair; -import java.security.KeyStore; -import java.security.PrivateKey; -import java.security.PublicKey; -import java.security.cert.X509Certificate; -import java.time.Duration; -import java.time.Instant; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Configures the JDisc https connector with the configserver's Athenz provider certificate and private key. - * - * @author bjorncs - */ -public class ConfigserverSslContextFactoryProvider extends TlsContextBasedProvider { - - private static final String CERTIFICATE_ALIAS = "athenz"; - private static final Duration EXPIRATION_MARGIN = Duration.ofHours(6); - private static final Path VESPA_SIA_DIRECTORY = Paths.get(Defaults.getDefaults().underVespaHome("var/vespa/sia")); - - private static final Logger log = Logger.getLogger(ConfigserverSslContextFactoryProvider.class.getName()); - - private final TlsContext tlsContext; - private final MutableX509KeyManager keyManager = new MutableX509KeyManager(); - private final ScheduledExecutorService scheduler = - Executors.newSingleThreadScheduledExecutor(runnable -> new Thread(runnable, "configserver-ssl-context-factory-provider")); - private final ZtsClient ztsClient; - private final KeyProvider keyProvider; - private final AthenzProviderServiceConfig athenzProviderServiceConfig; - private final AthenzService configserverIdentity; - - @Inject - public ConfigserverSslContextFactoryProvider(ServiceIdentityProvider bootstrapIdentity, - KeyProvider keyProvider, - AthenzProviderServiceConfig config) { - this.athenzProviderServiceConfig = config; - this.ztsClient = new DefaultZtsClient.Builder(URI.create(athenzProviderServiceConfig.ztsUrl())) - .withIdentityProvider(bootstrapIdentity).build(); - this.keyProvider = keyProvider; - this.configserverIdentity = new AthenzService(athenzProviderServiceConfig.domain(), athenzProviderServiceConfig.serviceName()); - - Duration updatePeriod = Duration.ofDays(config.updatePeriodDays()); - Path trustStoreFile = Paths.get(config.athenzCaTrustStore()); - this.tlsContext = createTlsContext(keyProvider, keyManager, trustStoreFile, updatePeriod, configserverIdentity, ztsClient, athenzProviderServiceConfig); - scheduler.scheduleAtFixedRate(new KeystoreUpdater(keyManager), - updatePeriod.toDays()/*initial delay*/, - updatePeriod.toDays(), - TimeUnit.DAYS); - } - - @Override - protected TlsContext getTlsContext(String containerId, int port) { - return tlsContext; - } - - Instant getCertificateNotAfter() { - return keyManager.currentManager().getCertificateChain(CERTIFICATE_ALIAS)[0].getNotAfter().toInstant(); - } - - @Override - public void deconstruct() { - try { - scheduler.shutdownNow(); - scheduler.awaitTermination(30, TimeUnit.SECONDS); - ztsClient.close(); - super.deconstruct(); - } catch (InterruptedException e) { - throw new RuntimeException("Failed to shutdown Athenz certificate updater on time", e); - } - } - - private static TlsContext createTlsContext(KeyProvider keyProvider, - MutableX509KeyManager keyManager, - Path trustStoreFile, - Duration updatePeriod, - AthenzService configserverIdentity, - ZtsClient ztsClient, - AthenzProviderServiceConfig zoneConfig) { - KeyStore keyStore = - tryReadKeystoreFile(configserverIdentity, updatePeriod) - .orElseGet(() -> updateKeystore(configserverIdentity, generateKeystorePassword(), keyProvider, ztsClient, zoneConfig)); - keyManager.updateKeystore(keyStore, new char[0]); - SSLContext sslContext = new SslContextBuilder() - .withTrustStore(trustStoreFile, KeyStoreType.JKS) - .withKeyManager(keyManager) - .build(); - return new DefaultTlsContext(sslContext, PeerAuthentication.WANT); - } - - private static Optional<KeyStore> tryReadKeystoreFile(AthenzService configserverIdentity, Duration updatePeriod) { - Optional<X509Certificate> certificate = SiaUtils.readCertificateFile(VESPA_SIA_DIRECTORY, configserverIdentity); - if (!certificate.isPresent()) return Optional.empty(); - Optional<PrivateKey> privateKey = SiaUtils.readPrivateKeyFile(VESPA_SIA_DIRECTORY, configserverIdentity); - if (!privateKey.isPresent()) return Optional.empty(); - Instant minimumExpiration = Instant.now().plus(updatePeriod).plus(EXPIRATION_MARGIN); - boolean isExpired = certificate.get().getNotAfter().toInstant().isBefore(minimumExpiration); - if (isExpired) return Optional.empty(); - KeyStore keyStore = KeyStoreBuilder.withType(KeyStoreType.JKS) - .withKeyEntry(CERTIFICATE_ALIAS, privateKey.get(), certificate.get()) - .build(); - return Optional.of(keyStore); - } - - private static KeyStore updateKeystore(AthenzService configserverIdentity, - char[] keystorePwd, - KeyProvider keyProvider, - ZtsClient ztsClient, - AthenzProviderServiceConfig zoneConfig) { - PrivateKey privateKey = keyProvider.getPrivateKey(zoneConfig.secretVersion()); - PublicKey publicKey = KeyUtils.extractPublicKey(privateKey); - Identity serviceIdentity = ztsClient.getServiceIdentity(configserverIdentity, - Integer.toString(zoneConfig.secretVersion()), - new KeyPair(publicKey, privateKey), - zoneConfig.certDnsSuffix()); - X509Certificate certificate = serviceIdentity.certificate(); - SiaUtils.writeCertificateFile(VESPA_SIA_DIRECTORY, configserverIdentity, certificate); - SiaUtils.writePrivateKeyFile(VESPA_SIA_DIRECTORY, configserverIdentity, privateKey); - Instant expirationTime = certificate.getNotAfter().toInstant(); - Duration expiry = Duration.between(certificate.getNotBefore().toInstant(), expirationTime); - log.log(Level.INFO, String.format("Got Athenz x509 certificate with expiry %s (expires %s)", expiry, expirationTime)); - return KeyStoreBuilder.withType(KeyStoreType.JKS) - .withKeyEntry(CERTIFICATE_ALIAS, privateKey, keystorePwd, certificate) - .build(); - } - - private static char[] generateKeystorePassword() { - return UUID.randomUUID().toString().toCharArray(); - } - - private class KeystoreUpdater implements Runnable { - final MutableX509KeyManager keyManager; - - KeystoreUpdater(MutableX509KeyManager keyManager) { - this.keyManager = keyManager; - } - - @Override - public void run() { - try { - log.log(Level.INFO, "Updating configserver provider certificate from ZTS"); - char[] keystorePwd = generateKeystorePassword(); - KeyStore keyStore = updateKeystore(configserverIdentity, keystorePwd, keyProvider, ztsClient, athenzProviderServiceConfig); - keyManager.updateKeystore(keyStore, keystorePwd); - log.log(Level.INFO, "Certificate successfully updated"); - } catch (Throwable t) { - log.log(Level.SEVERE, "Failed to update certificate from ZTS: " + t.getMessage(), t); - } - } - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGenerator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGenerator.java deleted file mode 100644 index 5143a38b2c1..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGenerator.java +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.config.provision.Zone; -import com.yahoo.net.HostName; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.identityprovider.api.ClusterType; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; -import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import com.yahoo.vespa.athenz.identityprovider.client.IdentityDocumentSigner; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.Allocation; - -import java.security.PrivateKey; -import java.time.Instant; -import java.util.HashSet; -import java.util.Set; - -/** - * Generates a signed identity document for a given hostname and type - * - * @author mortent - * @author bjorncs - */ -public class IdentityDocumentGenerator { - - private final IdentityDocumentSigner signer = new IdentityDocumentSigner(); - private final NodeRepository nodeRepository; - private final Zone zone; - private final KeyProvider keyProvider; - private final AthenzProviderServiceConfig athenzProviderServiceConfig; - - @Inject - public IdentityDocumentGenerator(AthenzProviderServiceConfig config, - NodeRepository nodeRepository, - Zone zone, - KeyProvider keyProvider) { - this.athenzProviderServiceConfig = config; - this.nodeRepository = nodeRepository; - this.zone = zone; - this.keyProvider = keyProvider; - } - - public SignedIdentityDocument generateSignedIdentityDocument(String hostname, IdentityType identityType) { - try { - Node node = nodeRepository.nodes().node(hostname).orElseThrow(() -> new RuntimeException("Unable to find node " + hostname)); - Allocation allocation = node.allocation().orElseThrow(() -> new RuntimeException("No allocation for node " + node.hostname())); - VespaUniqueInstanceId providerUniqueId = new VespaUniqueInstanceId( - allocation.membership().index(), - allocation.membership().cluster().id().value(), - allocation.owner().instance().value(), - allocation.owner().application().value(), - allocation.owner().tenant().value(), - zone.region().value(), - zone.environment().value(), - identityType); - - Set<String> ips = new HashSet<>(node.ipConfig().primary()); - - PrivateKey privateKey = keyProvider.getPrivateKey(athenzProviderServiceConfig.secretVersion()); - AthenzService providerService = new AthenzService(athenzProviderServiceConfig.domain(), athenzProviderServiceConfig.serviceName()); - - String configServerHostname = HostName.getLocalhost(); - Instant createdAt = Instant.now(); - var clusterType = ClusterType.from(allocation.membership().cluster().type().name()); - String signature = signer.generateSignature( - providerUniqueId, providerService, configServerHostname, - node.hostname(), createdAt, ips, identityType, privateKey); - return new SignedIdentityDocument( - signature, athenzProviderServiceConfig.secretVersion(), providerUniqueId, providerService, - SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION, configServerHostname, node.hostname(), - createdAt, ips, identityType, clusterType); - } catch (Exception e) { - throw new RuntimeException("Exception generating identity document: " + e.getMessage(), e); - } - } - -} - diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityProviderRequestHandler.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityProviderRequestHandler.java deleted file mode 100644 index c1dd70d7656..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityProviderRequestHandler.java +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.yahoo.component.annotation.Inject; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.restapi.RestApi; -import com.yahoo.restapi.RestApiException; -import com.yahoo.restapi.RestApiRequestHandler; -import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; -import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; - -import java.util.logging.Level; - -/** - * Handler implementing the Athenz Identity Provider API (Copper Argos). - * - * @author bjorncs - */ -public class IdentityProviderRequestHandler extends RestApiRequestHandler<IdentityProviderRequestHandler> { - - private final IdentityDocumentGenerator documentGenerator; - private final InstanceValidator instanceValidator; - - @Inject - public IdentityProviderRequestHandler(ThreadedHttpRequestHandler.Context context, - IdentityDocumentGenerator documentGenerator, - InstanceValidator instanceValidator) { - super(context, IdentityProviderRequestHandler::createRestApi); - this.documentGenerator = documentGenerator; - this.instanceValidator = instanceValidator; - } - - private static RestApi createRestApi(IdentityProviderRequestHandler self) { - return RestApi.builder() - .addRoute(RestApi.route("/athenz/v1/provider/identity-document/node/{host}") - .get(self::getNodeIdentityDocument)) - .addRoute(RestApi.route("/athenz/v1/provider/identity-document/tenant/{host}") - .get(self::getTenantIdentityDocument)) - .addRoute(RestApi.route("/athenz/v1/provider/instance") - .post(InstanceConfirmation.class, self::confirmInstance)) - .addRoute(RestApi.route("/athenz/v1/provider/refresh") - .post(InstanceConfirmation.class, self::confirmInstanceRefresh)) - .registerJacksonRequestEntity(InstanceConfirmation.class) - .registerJacksonResponseEntity(InstanceConfirmation.class) - .registerJacksonResponseEntity(SignedIdentityDocumentEntity.class) - // Overriding object mapper to change serialization of timestamps - .setObjectMapper(new ObjectMapper() - .registerModule(new JavaTimeModule()) - .registerModule(new Jdk8Module()) - .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true)) - .build(); - } - - private SignedIdentityDocumentEntity getNodeIdentityDocument(RestApi.RequestContext context) { - String host = context.pathParameters().getString("host").orElse(null); - return getIdentityDocument(host, IdentityType.NODE); - } - - private SignedIdentityDocumentEntity getTenantIdentityDocument(RestApi.RequestContext context) { - String host = context.pathParameters().getString("host").orElse(null); - return getIdentityDocument(host, IdentityType.TENANT); - } - - private InstanceConfirmation confirmInstance(RestApi.RequestContext context, InstanceConfirmation instanceConfirmation) { - log.log(Level.FINE, () -> instanceConfirmation.toString()); - if (!instanceValidator.isValidInstance(instanceConfirmation)) { - log.log(Level.SEVERE, "Invalid instance: " + instanceConfirmation); - throw new RestApiException.Forbidden("Instance is invalid"); - } - return instanceConfirmation; - } - - private InstanceConfirmation confirmInstanceRefresh(RestApi.RequestContext context, InstanceConfirmation instanceConfirmation) { - log.log(Level.FINE, () -> instanceConfirmation.toString()); - if (!instanceValidator.isValidRefresh(instanceConfirmation)) { - log.log(Level.SEVERE, "Invalid instance refresh: " + instanceConfirmation); - throw new RestApiException.Forbidden("Instance is invalid"); - } - return instanceConfirmation; - } - - private SignedIdentityDocumentEntity getIdentityDocument(String hostname, IdentityType identityType) { - if (hostname == null) { - throw new RestApiException.BadRequest("The 'hostname' query parameter is missing"); - } - try { - return EntityBindingsMapper.toSignedIdentityDocumentEntity(documentGenerator.generateSignedIdentityDocument(hostname, identityType)); - } catch (Exception e) { - String message = String.format("Unable to generate identity document for '%s': %s", hostname, e.getMessage()); - log.log(Level.SEVERE, message, e); - throw new RestApiException.InternalServerError(message, e); - } - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceConfirmation.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceConfirmation.java deleted file mode 100644 index 6c09a35ee3d..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceConfirmation.java +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonUnwrapped; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; - -/** - * InstanceConfirmation object as per Athenz InstanceConfirmation API. - * - * @author bjorncs - */ -public class InstanceConfirmation { - - @JsonProperty("provider") public final String provider; - @JsonProperty("domain") public final String domain; - @JsonProperty("service") public final String service; - - @JsonProperty("attestationData") @JsonSerialize(using = SignedIdentitySerializer.class) - public final SignedIdentityDocumentEntity signedIdentityDocument; - @JsonUnwrapped public final Map<String, String> attributes = new HashMap<>(); // optional attributes that Athenz may provide - - @JsonCreator - public InstanceConfirmation(@JsonProperty("provider") String provider, - @JsonProperty("domain") String domain, - @JsonProperty("service") String service, - @JsonProperty("attestationData") @JsonDeserialize(using = SignedIdentityDeserializer.class) - SignedIdentityDocumentEntity signedIdentityDocument) { - this.provider = provider; - this.domain = domain; - this.service = service; - this.signedIdentityDocument = signedIdentityDocument; - } - - @JsonAnySetter - public void set(String name, String value) { - attributes.put(name, value); - } - - @Override - public String toString() { - return "InstanceConfirmation{" + - "provider='" + provider + '\'' + - ", domain='" + domain + '\'' + - ", service='" + service + '\'' + - ", signedIdentityDocument='" + signedIdentityDocument + '\'' + - ", attributes=" + attributes + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - InstanceConfirmation that = (InstanceConfirmation) o; - return Objects.equals(provider, that.provider) && - Objects.equals(domain, that.domain) && - Objects.equals(service, that.service) && - Objects.equals(signedIdentityDocument, that.signedIdentityDocument) && - Objects.equals(attributes, that.attributes); - } - - @Override - public int hashCode() { - return Objects.hash(provider, domain, service, signedIdentityDocument, attributes); - } - - public static class SignedIdentityDeserializer extends JsonDeserializer<SignedIdentityDocumentEntity> { - @Override - public SignedIdentityDocumentEntity deserialize( - JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { - String value = jsonParser.getValueAsString(); - return Utils.getMapper().readValue(value, SignedIdentityDocumentEntity.class); - } - } - - public static class SignedIdentitySerializer extends JsonSerializer<SignedIdentityDocumentEntity> { - @Override - public void serialize( - SignedIdentityDocumentEntity document, JsonGenerator gen, SerializerProvider serializers) throws IOException { - gen.writeString(Utils.getMapper().writeValueAsString(document)); - } - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidator.java deleted file mode 100644 index d8bbf743d8c..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidator.java +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.google.common.net.InetAddresses; -import com.yahoo.component.annotation.Inject; -import com.yahoo.config.model.api.ApplicationInfo; -import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.config.model.api.SuperModelProvider; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.identityprovider.api.ClusterType; -import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; -import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import com.yahoo.vespa.athenz.identityprovider.client.IdentityDocumentSigner; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; - -import java.net.InetAddress; -import java.net.URI; -import java.security.PublicKey; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.function.Supplier; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Verifies that the instance's identity document is valid - * - * @author bjorncs - * @author mortent - */ -public class InstanceValidator { - - private static final Logger log = Logger.getLogger(InstanceValidator.class.getName()); - static final String SERVICE_PROPERTIES_DOMAIN_KEY = "identity.domain"; - static final String SERVICE_PROPERTIES_SERVICE_KEY = "identity.service"; - static final String INSTANCE_ID_DELIMITER = ".instanceid.athenz."; - - public static final String SAN_IPS_ATTRNAME = "sanIP"; - public static final String SAN_DNS_ATTRNAME = "sanDNS"; - public static final String SAN_URI_ATTRNAME = "sanURI"; - - private final AthenzService tenantDockerContainerIdentity; - private final IdentityDocumentSigner signer; - private final KeyProvider keyProvider; - private final SuperModelProvider superModelProvider; - private final NodeRepository nodeRepository; - - @Inject - public InstanceValidator(KeyProvider keyProvider, - SuperModelProvider superModelProvider, - NodeRepository nodeRepository, - AthenzProviderServiceConfig config) { - this(keyProvider, superModelProvider, nodeRepository, new IdentityDocumentSigner(), new AthenzService(config.tenantService())); - } - - public InstanceValidator(KeyProvider keyProvider, - SuperModelProvider superModelProvider, - NodeRepository nodeRepository, - IdentityDocumentSigner identityDocumentSigner, - AthenzService tenantIdentity){ - this.keyProvider = keyProvider; - this.superModelProvider = superModelProvider; - this.nodeRepository = nodeRepository; - this.signer = identityDocumentSigner; - this.tenantDockerContainerIdentity = tenantIdentity; - } - - public boolean isValidInstance(InstanceConfirmation instanceConfirmation) { - try { - validateInstance(instanceConfirmation); - return true; - } catch (ValidationException e) { - log.log(e.logLevel(), e.messageSupplier()); - return false; - } - } - - public void validateInstance(InstanceConfirmation req) throws ValidationException { - SignedIdentityDocument signedIdentityDocument = EntityBindingsMapper.toSignedIdentityDocument(req.signedIdentityDocument); - VespaUniqueInstanceId providerUniqueId = signedIdentityDocument.providerUniqueId(); - ApplicationId applicationId = ApplicationId.from( - providerUniqueId.tenant(), providerUniqueId.application(), providerUniqueId.instance()); - - VespaUniqueInstanceId csrProviderUniqueId = getVespaUniqueInstanceId(req); - if(! providerUniqueId.equals(csrProviderUniqueId)) { - var msg = String.format("Instance %s has invalid provider unique ID in CSR (%s)", providerUniqueId, csrProviderUniqueId); - throw new ValidationException(Level.WARNING, () -> msg); - } - - if (! isSameIdentityAsInServicesXml(applicationId, req.domain, req.service)) { - Supplier<String> msg = () -> "Invalid identity '%s.%s' in services.xml".formatted(req.domain, req.service); - throw new ValidationException(Level.FINE, msg); - } - - log.log(Level.FINE, () -> String.format("Validating instance %s.", providerUniqueId)); - - PublicKey publicKey = keyProvider.getPublicKey(signedIdentityDocument.signingKeyVersion()); - if (! signer.hasValidSignature(signedIdentityDocument, publicKey)) { - var msg = String.format("Instance %s has invalid signature.", providerUniqueId); - throw new ValidationException(Level.SEVERE, () -> msg); - } - - validateAttributes(req, providerUniqueId); - log.log(Level.FINE, () -> String.format("Instance %s is valid.", providerUniqueId)); - } - - // TODO Add actual validation. Cannot reuse isValidInstance as identity document is not part of the refresh request. - // We'll have to perform some validation on the instance id and other fields of the attribute map. - // Separate between tenant and node certificate as well. - public boolean isValidRefresh(InstanceConfirmation confirmation) { - log.log(Level.FINE, () -> String.format("Accepting refresh for instance with identity '%s', provider '%s', instanceId '%s'.", - new AthenzService(confirmation.domain, confirmation.service).getFullName(), - confirmation.provider, - confirmation.attributes.get(SAN_DNS_ATTRNAME))); - try { - validateAttributes(confirmation, getVespaUniqueInstanceId(confirmation)); - return true; - } catch (ValidationException e) { - log.log(e.logLevel(), e.messageSupplier()); - return false; - } catch (Exception e) { - log.log(Level.WARNING, "Encountered exception while refreshing certificate for confirmation: " + confirmation, e); - return false; - } - } - - private VespaUniqueInstanceId getVespaUniqueInstanceId(InstanceConfirmation instanceConfirmation) { - // Find a list of SAN DNS - List<String> sanDNS = Optional.ofNullable(instanceConfirmation.attributes.get(SAN_DNS_ATTRNAME)) - .map(s -> s.split(",")) - .map(Arrays::asList).stream().flatMap(Collection::stream).toList(); - - return sanDNS.stream() - .filter(dns -> dns.contains(INSTANCE_ID_DELIMITER)) - .findFirst() - .map(s -> s.replaceAll(INSTANCE_ID_DELIMITER + ".*", "")) - .map(VespaUniqueInstanceId::fromDottedString) - .orElse(null); - } - - private void validateAttributes(InstanceConfirmation confirmation, VespaUniqueInstanceId vespaUniqueInstanceId) - throws ValidationException { - if(vespaUniqueInstanceId == null) { - var msg = "Unable to find unique instance ID in refresh request: " + confirmation.toString(); - throw new ValidationException(Level.WARNING, () -> msg); - } - - // Find node matching vespa unique id - Node node = nodeRepository.nodes().list().stream() - .filter(n -> n.allocation().isPresent()) - .filter(n -> nodeMatchesVespaUniqueId(n, vespaUniqueInstanceId)) - .findFirst() // Should be only one - .orElse(null); - if(node == null) { - var msg = "Invalid InstanceConfirmation, No nodes matching uniqueId: " + vespaUniqueInstanceId; - throw new ValidationException(Level.WARNING, () -> msg); - } - - // Find list of ipaddresses - List<InetAddress> ips = Optional.ofNullable(confirmation.attributes.get(SAN_IPS_ATTRNAME)) - .map(s -> s.split(",")) - .map(Arrays::asList).stream().flatMap(Collection::stream) - .map(InetAddresses::forString) - .toList(); - - List<InetAddress> nodeIpAddresses = node.ipConfig().primary().stream() - .map(InetAddresses::forString) - .toList(); - - // Validate that ipaddresses in request are valid for node - - if(! nodeIpAddresses.containsAll(ips)) { - var msg = "Invalid InstanceConfirmation, wrong ip in : " + vespaUniqueInstanceId; - throw new ValidationException(Level.WARNING, () -> msg); - } - - var urisCommaSeparated = confirmation.attributes.get(SAN_URI_ATTRNAME); - Set<URI> requestedUris; - try { - requestedUris = Optional.ofNullable(urisCommaSeparated).stream() - .flatMap(s -> Arrays.stream(s.split(","))).map(URI::create).collect(Collectors.toSet()); - } catch (IllegalArgumentException e) { - throw new ValidationException(Level.WARNING, () -> "Invalid SAN URIs: " + urisCommaSeparated, e); - } - var clusterType = node.allocation().map(a -> a.membership().cluster().type()).orElse(null); - Set<URI> allowedUris = clusterType != null - ? Set.of(ClusterType.from(clusterType.name()).asCertificateSanUri()) : Set.of(); - if (!allowedUris.containsAll(requestedUris)) { - Supplier<String> msg = () -> "Illegal SAN URIs: expected '%s' found '%s'".formatted(allowedUris, requestedUris); - throw new ValidationException(Level.WARNING, msg); - } - } - - private boolean nodeMatchesVespaUniqueId(Node node, VespaUniqueInstanceId vespaUniqueInstanceId) { - return node.allocation().map(allocation -> - allocation.membership().index() == vespaUniqueInstanceId.clusterIndex() && - allocation.membership().cluster().id().value().equals(vespaUniqueInstanceId.clusterId()) && - allocation.owner().instance().value().equals(vespaUniqueInstanceId.instance()) && - allocation.owner().application().value().equals(vespaUniqueInstanceId.application()) && - allocation.owner().tenant().value().equals(vespaUniqueInstanceId.tenant())) - .orElse(false); - } - - // If/when we don't care about logging exactly whats wrong, this can be simplified - // TODO Use identity type to determine if this check should be performed - private boolean isSameIdentityAsInServicesXml(ApplicationId applicationId, String domain, String service) { - - Optional<ApplicationInfo> applicationInfo = superModelProvider.getSuperModel().getApplicationInfo(applicationId); - - if (applicationInfo.isEmpty()) { - log.info(String.format("Could not find application info for %s, existing applications: %s", - applicationId.serializedForm(), - superModelProvider.getSuperModel().getAllApplicationInfos())); - return false; - } - - if (tenantDockerContainerIdentity.equals(new AthenzService(domain, service))) { - return true; - } - - Optional<ServiceInfo> matchingServiceInfo = applicationInfo.get() - .getModel() - .getHosts() - .stream() - .flatMap(hostInfo -> hostInfo.getServices().stream()) - .filter(serviceInfo -> serviceInfo.getProperty(SERVICE_PROPERTIES_DOMAIN_KEY).isPresent()) - .filter(serviceInfo -> serviceInfo.getProperty(SERVICE_PROPERTIES_SERVICE_KEY).isPresent()) - .findFirst(); - - if (matchingServiceInfo.isEmpty()) { - log.info(String.format("Application %s has not specified domain/service", applicationId.serializedForm())); - return false; - } - - String domainInConfig = matchingServiceInfo.get().getProperty(SERVICE_PROPERTIES_DOMAIN_KEY).get(); - String serviceInConfig = matchingServiceInfo.get().getProperty(SERVICE_PROPERTIES_SERVICE_KEY).get(); - if (!domainInConfig.equals(domain) || !serviceInConfig.equals(service)) { - log.warning(String.format("domain '%s' or service '%s' does not match the one in config for application %s", - domain, service, applicationId.serializedForm())); - return false; - } - - return true; - } - - public static class ValidationException extends Exception { - private final Level logLevel; - private final Supplier<String> msg; - - public ValidationException(Level logLevel, Supplier<String> msg) { this(logLevel, msg, null); } - public ValidationException(Level logLevel, Supplier<String> msg, Throwable cause) { super(cause); this.logLevel = logLevel; this.msg = msg; } - - @Override public String getMessage() { return msg.get(); } - public Level logLevel() { return logLevel; } - public Supplier<String> messageSupplier() { return msg; } - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/KeyProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/KeyProvider.java deleted file mode 100644 index 324f927fd73..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/KeyProvider.java +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import java.security.KeyPair; -import java.security.PrivateKey; -import java.security.PublicKey; - -/** - * @author bjorncs - */ -public interface KeyProvider { - PrivateKey getPrivateKey(int version); - - PublicKey getPublicKey(int version); - - default KeyPair getKeyPair(int version) { - return new KeyPair(getPublicKey(version), getPrivateKey(version)); - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/Utils.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/Utils.java deleted file mode 100644 index 5c4942f37cb..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/Utils.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; - -/** - * @author bjorncs - */ -public class Utils { - - private static final ObjectMapper mapper = createObjectMapper(); - - public static ObjectMapper getMapper() { - return mapper; - } - - private static ObjectMapper createObjectMapper() { - ObjectMapper mapper = new ObjectMapper(); - mapper.registerModule(new JavaTimeModule()); - return mapper; - } - -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/package-info.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/package-info.java deleted file mode 100644 index 0cb5c9d4f82..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author bjorncs - */ -@ExportPackage -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/Certificates.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/Certificates.java deleted file mode 100644 index df904bf8010..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/Certificates.java +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca; - -import com.yahoo.security.Pkcs10Csr; -import com.yahoo.security.SubjectAlternativeName; -import com.yahoo.security.X509CertificateBuilder; -import com.yahoo.security.X509CertificateUtils; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; - -import java.security.PrivateKey; -import java.security.cert.X509Certificate; -import java.time.Clock; -import java.time.Duration; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.stream.Collectors; - -import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; -import static com.yahoo.security.SubjectAlternativeName.Type.DNS; - -/** - * Helper class for creating {@link X509Certificate}s. - * - * @author mpolden - */ -public class Certificates { - - private static final Duration CERTIFICATE_TTL = Duration.ofDays(30); - private static final String INSTANCE_ID_DELIMITER = ".instanceid.athenz."; - - private final Clock clock; - - public Certificates(Clock clock) { - this.clock = Objects.requireNonNull(clock, "clock must be non-null"); - } - - /** Create a new certificate from csr signed by the given CA private key */ - public X509Certificate create(Pkcs10Csr csr, X509Certificate caCertificate, PrivateKey caPrivateKey) { - var x500principal = caCertificate.getSubjectX500Principal(); - var now = clock.instant(); - var notBefore = now.minus(Duration.ofHours(1)); - var notAfter = now.plus(CERTIFICATE_TTL); - var builder = X509CertificateBuilder.fromCsr(csr, - x500principal, - notBefore, - notAfter, - caPrivateKey, - SHA256_WITH_ECDSA, - X509CertificateBuilder.generateRandomSerialNumber()); - for (var san : csr.getSubjectAlternativeNames()) { - builder = builder.addSubjectAlternativeName(san.decode()); - } - return builder.build(); - } - - /** Returns instance ID parsed from the Subject Alternative Names in given csr */ - public static String instanceIdFrom(Pkcs10Csr csr) { - return getInstanceIdFromSAN(csr.getSubjectAlternativeNames()) - .orElseThrow(() -> new IllegalArgumentException("No instance ID found in CSR")); - } - - public static Optional<String> instanceIdFrom(X509Certificate certificate) { - return getInstanceIdFromSAN(X509CertificateUtils.getSubjectAlternativeNames(certificate)); - } - - private static Optional<String> getInstanceIdFromSAN(List<SubjectAlternativeName> subjectAlternativeNames) { - return subjectAlternativeNames.stream() - .filter(san -> san.getType() == DNS) - .map(SubjectAlternativeName::getValue) - .map(Certificates::parseInstanceId) - .flatMap(Optional::stream) - .map(VespaUniqueInstanceId::asDottedString) - .findFirst(); - } - - private static Optional<VespaUniqueInstanceId> parseInstanceId(String dnsName) { - var delimiterStart = dnsName.indexOf(INSTANCE_ID_DELIMITER); - if (delimiterStart == -1) return Optional.empty(); - dnsName = dnsName.substring(0, delimiterStart); - try { - return Optional.of(VespaUniqueInstanceId.fromDottedString(dnsName)); - } catch (IllegalArgumentException e) { - return Optional.empty(); - } - } - - public static String getSubjectAlternativeNames(Pkcs10Csr csr, SubjectAlternativeName.Type sanType) { - return csr.getSubjectAlternativeNames().stream() - .map(SubjectAlternativeName::decode) - .filter(san -> san.getType() == sanType) - .map(SubjectAlternativeName::getValue) - .collect(Collectors.joining(",")); - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceIdentity.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceIdentity.java deleted file mode 100644 index f33ec4fbd6d..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceIdentity.java +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.instance; - -import java.security.cert.X509Certificate; -import java.util.Objects; -import java.util.Optional; - -/** - * A signed instance identity object that includes a client certificate. This is the result of a successful - * {@link InstanceRegistration} and is the same type as InstanceIdentity in the ZTS API. - * - * @author mpolden - */ -public class InstanceIdentity { - - private final String provider; - private final String service; - private final String instanceId; - private final Optional<X509Certificate> x509Certificate; - - public InstanceIdentity(String provider, String service, String instanceId, Optional<X509Certificate> x509Certificate) { - this.provider = Objects.requireNonNull(provider, "provider must be non-null"); - this.service = Objects.requireNonNull(service, "service must be non-null"); - this.instanceId = Objects.requireNonNull(instanceId, "instanceId must be non-null"); - this.x509Certificate = Objects.requireNonNull(x509Certificate, "x509Certificate must be non-null"); - } - - /** Same as {@link InstanceRegistration#domain()} */ - public String provider() { - return provider; - } - - /** Same as {@link InstanceRegistration#service()} ()} */ - public String service() { - return service; - } - - /** A unique identifier of the instance to which the certificate is issued */ - public String instanceId() { - return instanceId; - } - - /** The issued certificate */ - public Optional<X509Certificate> x509Certificate() { - return x509Certificate; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - InstanceIdentity that = (InstanceIdentity) o; - return provider.equals(that.provider) && - service.equals(that.service) && - instanceId.equals(that.instanceId) && - x509Certificate.equals(that.x509Certificate); - } - - @Override - public int hashCode() { - return Objects.hash(provider, service, instanceId, x509Certificate); - } - -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRefresh.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRefresh.java deleted file mode 100644 index d63ee7f979f..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRefresh.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.instance; - -import com.yahoo.security.Pkcs10Csr; - -import java.util.Objects; - -/** - * Information for refreshing a instance in the system. This is the same type as InstanceRefreshInformation type in - * the ZTS API. - * - * @author mpolden - */ -public class InstanceRefresh { - - private final Pkcs10Csr csr; - - public InstanceRefresh(Pkcs10Csr csr) { - this.csr = Objects.requireNonNull(csr, "csr must be non-null"); - } - - /** The Certificate Signed Request describing the wanted certificate */ - public Pkcs10Csr csr() { - return csr; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - InstanceRefresh that = (InstanceRefresh) o; - return csr.equals(that.csr); - } - - @Override - public int hashCode() { - return Objects.hash(csr); - } - -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRegistration.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRegistration.java deleted file mode 100644 index 231954976bf..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRegistration.java +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.instance; - -import com.yahoo.security.Pkcs10Csr; -import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; - -import java.util.Objects; - -/** - * Information for registering a new instance in the system. This is the same type as InstanceRegisterInformation type - * in the ZTS API. - * - * @author mpolden - */ -public class InstanceRegistration { - - private final String provider; - private final String domain; - private final String service; - private final SignedIdentityDocument attestationData; - private final Pkcs10Csr csr; - - public InstanceRegistration(String provider, String domain, String service, SignedIdentityDocument attestationData, Pkcs10Csr csr) { - this.provider = Objects.requireNonNull(provider, "provider must be non-null"); - this.domain = Objects.requireNonNull(domain, "domain must be non-null"); - this.service = Objects.requireNonNull(service, "service must be non-null"); - this.attestationData = Objects.requireNonNull(attestationData, "attestationData must be non-null"); - this.csr = Objects.requireNonNull(csr, "csr must be non-null"); - } - - /** The provider which issued the attestation data contained in this */ - public String provider() { - return provider; - } - - /** Athenz domain of the instance */ - public String domain() { - return domain; - } - - /** Athenz service of the instance */ - public String service() { - return service; - } - - /** Host document describing this instance (received from config server) */ - public SignedIdentityDocument attestationData() { - return attestationData; - } - - /** The Certificate Signed Request describing the wanted certificate */ - public Pkcs10Csr csr() { - return csr; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - InstanceRegistration that = (InstanceRegistration) o; - return provider.equals(that.provider) && - domain.equals(that.domain) && - service.equals(that.service) && - attestationData.equals(that.attestationData) && - csr.equals(that.csr); - } - - @Override - public int hashCode() { - return Objects.hash(provider, domain, service, attestationData, csr); - } - - @Override - public String toString() { - return "InstanceRegistration{" + - "provider='" + provider + '\'' + - ", domain='" + domain + '\'' + - ", service='" + service + '\'' + - ", attestationData='" + attestationData.toString() + '\'' + - ", csr=" + csr.toString() + - '}'; - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java deleted file mode 100644 index 531a815922b..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.restapi; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.jdisc.http.server.jetty.RequestUtils; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.Path; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.SubjectAlternativeName; -import com.yahoo.security.X509CertificateUtils; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceConfirmation; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceValidator; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; -import com.yahoo.vespa.hosted.ca.Certificates; -import com.yahoo.vespa.hosted.ca.instance.InstanceIdentity; -import com.yahoo.vespa.hosted.ca.instance.InstanceRefresh; -import com.yahoo.yolean.Exceptions; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.security.PrivateKey; -import java.security.cert.X509Certificate; -import java.time.Clock; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.function.Function; -import java.util.logging.Level; - -/** - * REST API for issuing and refreshing node certificates in a hosted Vespa system. - * - * The API implements the following subset of methods from the Athenz ZTS REST API: - * - * - Instance registration - * - Instance refresh - * - * @author mpolden - */ -public class CertificateAuthorityApiHandler extends ThreadedHttpRequestHandler { - - private final SecretStore secretStore; - private final Certificates certificates; - private final String caPrivateKeySecretName; - private final String caCertificateSecretName; - private final InstanceValidator instanceValidator; - - @Inject - public CertificateAuthorityApiHandler(Context ctx, SecretStore secretStore, AthenzProviderServiceConfig athenzProviderServiceConfig, InstanceValidator instanceValidator) { - this(ctx, secretStore, new Certificates(Clock.systemUTC()), athenzProviderServiceConfig, instanceValidator); - } - - CertificateAuthorityApiHandler(Context ctx, SecretStore secretStore, Certificates certificates, AthenzProviderServiceConfig athenzProviderServiceConfig, InstanceValidator instanceValidator) { - super(ctx); - this.secretStore = secretStore; - this.certificates = certificates; - this.caPrivateKeySecretName = athenzProviderServiceConfig.secretName(); - this.caCertificateSecretName = athenzProviderServiceConfig.caCertSecretName(); - this.instanceValidator = instanceValidator; - } - - @Override - public HttpResponse handle(HttpRequest request) { - try { - switch (request.getMethod()) { - case POST: return handlePost(request); - default: return ErrorResponse.methodNotAllowed("Method " + request.getMethod() + " is unsupported"); - } - } catch (IllegalArgumentException e) { - return ErrorResponse.badRequest(request.getMethod() + " " + request.getUri() + " failed: " + Exceptions.toMessageString(e)); - } catch (RuntimeException e) { - log.log(Level.WARNING, "Unexpected error handling " + request.getMethod() + " " + request.getUri(), e); - return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); - } - } - - private HttpResponse handlePost(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/ca/v1/instance/")) return registerInstance(request); - if (path.matches("/ca/v1/instance/{provider}/{domain}/{service}/{instanceId}")) return refreshInstance(request, path.get("provider"), path.get("service"), path.get("instanceId")); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse registerInstance(HttpRequest request) { - var instanceRegistration = deserializeRequest(request, InstanceSerializer::registrationFromSlime); - - InstanceConfirmation confirmation = new InstanceConfirmation(instanceRegistration.provider(), instanceRegistration.domain(), instanceRegistration.service(), EntityBindingsMapper.toSignedIdentityDocumentEntity(instanceRegistration.attestationData())); - confirmation.set(InstanceValidator.SAN_IPS_ATTRNAME, Certificates.getSubjectAlternativeNames(instanceRegistration.csr(), SubjectAlternativeName.Type.IP)); - confirmation.set(InstanceValidator.SAN_DNS_ATTRNAME, Certificates.getSubjectAlternativeNames(instanceRegistration.csr(), SubjectAlternativeName.Type.DNS)); - if (!instanceValidator.isValidInstance(confirmation)) { - log.log(Level.INFO, "Invalid instance registration for " + instanceRegistration.toString()); - return ErrorResponse.forbidden("Unable to launch service: " +instanceRegistration.service()); - } - var certificate = certificates.create(instanceRegistration.csr(), caCertificate(), caPrivateKey()); - var instanceId = Certificates.instanceIdFrom(instanceRegistration.csr()); - var identity = new InstanceIdentity(instanceRegistration.provider(), instanceRegistration.service(), instanceId, - Optional.of(certificate)); - return new SlimeJsonResponse(InstanceSerializer.identityToSlime(identity)); - } - - private HttpResponse refreshInstance(HttpRequest request, String provider, String service, String instanceId) { - var instanceRefresh = deserializeRequest(request, InstanceSerializer::refreshFromSlime); - var instanceIdFromCsr = Certificates.instanceIdFrom(instanceRefresh.csr()); - - var athenzService = getRequestAthenzService(request); - - if (!instanceIdFromCsr.equals(instanceId)) { - throw new IllegalArgumentException("Mismatch between instance ID in URL path and instance ID in CSR " + - "[instanceId=" + instanceId + ",instanceIdFromCsr=" + instanceIdFromCsr + - "]"); - } - - // Verify that the csr instance id matches one of the certificates in the chain - refreshesSameInstanceId(instanceIdFromCsr, request); - - - // Validate that there is no privilege escalation (can only refresh same service) - refreshesSameService(instanceRefresh, athenzService); - - InstanceConfirmation instanceConfirmation = new InstanceConfirmation(provider, athenzService.getDomain().getName(), athenzService.getName(), null); - instanceConfirmation.set(InstanceValidator.SAN_IPS_ATTRNAME, Certificates.getSubjectAlternativeNames(instanceRefresh.csr(), SubjectAlternativeName.Type.IP)); - instanceConfirmation.set(InstanceValidator.SAN_DNS_ATTRNAME, Certificates.getSubjectAlternativeNames(instanceRefresh.csr(), SubjectAlternativeName.Type.DNS)); - if(!instanceValidator.isValidRefresh(instanceConfirmation)) { - return ErrorResponse.forbidden("Unable to refresh cert: " + instanceRefresh.csr().getSubject().toString()); - } - - var certificate = certificates.create(instanceRefresh.csr(), caCertificate(), caPrivateKey()); - var identity = new InstanceIdentity(provider, service, instanceIdFromCsr, Optional.of(certificate)); - return new SlimeJsonResponse(InstanceSerializer.identityToSlime(identity)); - } - - public void refreshesSameInstanceId(String csrInstanceId, HttpRequest request) { - String certificateInstanceId = getRequestCertificateChain(request).stream() - .map(Certificates::instanceIdFrom) - .filter(Optional::isPresent) - .map(Optional::get) - .findAny().orElseThrow(() -> new IllegalArgumentException("No client certificate with instance id in request.")); - - if(! Objects.equals(certificateInstanceId, csrInstanceId)) { - throw new IllegalArgumentException("Mismatch between instance ID in client certificate and instance ID in CSR " + - "[instanceId=" + certificateInstanceId + ",instanceIdFromCsr=" + csrInstanceId + - "]"); - } - } - - private void refreshesSameService(InstanceRefresh instanceRefresh, AthenzService athenzService) { - List<String> commonNames = X509CertificateUtils.getCommonNames(instanceRefresh.csr().getSubject()); - if(commonNames.size() != 1 && !Objects.equals(commonNames.get(0), athenzService.getFullName())) { - throw new IllegalArgumentException(String.format("Invalid request, trying to refresh service %s using service %s.", instanceRefresh.csr().getSubject().getName(), athenzService.getFullName())); - } - } - - /** Returns CA certificate from secret store */ - private X509Certificate caCertificate() { - return X509CertificateUtils.fromPem(secretStore.getSecret(caCertificateSecretName)); - } - - private List<X509Certificate> getRequestCertificateChain(HttpRequest request) { - return Optional.ofNullable(request.getJDiscRequest().context().get(RequestUtils.JDISC_REQUEST_X509CERT)) - .map(X509Certificate[].class::cast) - .map(Arrays::asList) - .orElse(Collections.emptyList()); - } - - private AthenzService getRequestAthenzService(HttpRequest request) { - return getRequestCertificateChain(request).stream() - .findFirst() - .flatMap(X509CertificateUtils::getSubjectCommonName) - .map(AthenzService::new) - .orElseThrow(() -> new RuntimeException("No certificate found")); - } - - /** Returns CA private key from secret store */ - private PrivateKey caPrivateKey() { - return KeyUtils.fromPemEncodedPrivateKey(secretStore.getSecret(caPrivateKeySecretName)); - } - - private static <T> T deserializeRequest(HttpRequest request, Function<Slime, T> serializer) { - try { - var slime = SlimeUtils.jsonToSlime(request.getData().readAllBytes()); - return serializer.apply(slime); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java deleted file mode 100644 index fec03afab69..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.restapi; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.yahoo.security.Pkcs10CsrUtils; -import com.yahoo.security.X509CertificateUtils; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.text.StringUtilities; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.identityprovider.api.ClusterType; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; -import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import com.yahoo.vespa.hosted.ca.instance.InstanceIdentity; -import com.yahoo.vespa.hosted.ca.instance.InstanceRefresh; -import com.yahoo.vespa.hosted.ca.instance.InstanceRegistration; - -import java.io.IOException; -import java.time.Instant; -import java.util.HashSet; -import java.util.Set; - -/** - * @author mpolden - */ -public class InstanceSerializer { - - private static final String PROVIDER_FIELD = "provider"; - private static final String DOMAIN_FIELD = "domain"; - private static final String SERVICE_FIELD = "service"; - private static final String ATTESTATION_DATA_FIELD = "attestationData"; - private static final String CSR_FIELD = "csr"; - private static final String NAME_FIELD = "service"; - private static final String INSTANCE_ID_FIELD = "instanceId"; - private static final String X509_CERTIFICATE_FIELD = "x509Certificate"; - - private static final String IDD_SIGNATURE_FIELD = "signature"; - private static final String IDD_SIGNING_KEY_VERSION_FIELD = "signing-key-version"; - private static final String IDD_PROVIDER_UNIQUE_ID_FIELD = "provider-unique-id"; - private static final String IDD_PROVIDER_SERVICE_FIELD = "provider-service"; - private static final String IDD_DOCUMENT_VERSION_FIELD = "document-version"; - private static final String IDD_CONFIGSERVER_HOSTNAME_FIELD = "configserver-hostname"; - private static final String IDD_INSTANCE_HOSTNAME_FIELD = "instance-hostname"; - private static final String IDD_CREATED_AT_FIELD = "created-at"; - private static final String IDD_IPADDRESSES_FIELD = "ip-addresses"; - private static final String IDD_IDENTITY_TYPE_FIELD = "identity-type"; - private static final String IDD_CLUSTER_TYPE_FIELD = "cluster-type"; - - private static final ObjectMapper objectMapper = new ObjectMapper(); - static { - objectMapper.registerModule(new JavaTimeModule()); - } - - private InstanceSerializer() {} - - public static InstanceRegistration registrationFromSlime(Slime slime) { - Cursor root = slime.get(); - return new InstanceRegistration(requireField(PROVIDER_FIELD, root).asString(), - requireField(DOMAIN_FIELD, root).asString(), - requireField(SERVICE_FIELD, root).asString(), - attestationDataToIdentityDocument(StringUtilities.unescape(requireField(ATTESTATION_DATA_FIELD, root).asString())), - Pkcs10CsrUtils.fromPem(requireField(CSR_FIELD, root).asString())); - } - - public static InstanceRefresh refreshFromSlime(Slime slime) { - Cursor root = slime.get(); - return new InstanceRefresh(Pkcs10CsrUtils.fromPem(requireField(CSR_FIELD, root).asString())); - } - - public static Slime identityToSlime(InstanceIdentity identity) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString(PROVIDER_FIELD, identity.provider()); - root.setString(NAME_FIELD, identity.service()); - root.setString(INSTANCE_ID_FIELD, identity.instanceId()); - identity.x509Certificate() - .map(X509CertificateUtils::toPem) - .ifPresent(pem -> root.setString(X509_CERTIFICATE_FIELD, pem)); - return slime; - } - - public static SignedIdentityDocument attestationDataToIdentityDocument(String attestationData) { - Slime slime = SlimeUtils.jsonToSlime(attestationData); - Cursor root = slime.get(); - String signature = requireField(IDD_SIGNATURE_FIELD, root).asString(); - long signingKeyVersion = requireField(IDD_SIGNING_KEY_VERSION_FIELD, root).asLong(); - VespaUniqueInstanceId providerUniqueId = VespaUniqueInstanceId.fromDottedString(requireField(IDD_PROVIDER_UNIQUE_ID_FIELD, root).asString()); - AthenzService athenzService = new AthenzService(requireField(IDD_PROVIDER_SERVICE_FIELD, root).asString()); - long documentVersion = requireField(IDD_DOCUMENT_VERSION_FIELD, root).asLong(); - String configserverHostname = requireField(IDD_CONFIGSERVER_HOSTNAME_FIELD, root).asString(); - String instanceHostname = requireField(IDD_INSTANCE_HOSTNAME_FIELD, root).asString(); - double createdAtTimestamp = requireField(IDD_CREATED_AT_FIELD, root).asDouble(); - Instant createdAt = getJsr310Instant(createdAtTimestamp); - Set<String> ips = new HashSet<>(); - requireField(IDD_IPADDRESSES_FIELD, root).traverse((ArrayTraverser) (__, entry) -> ips.add(entry.asString())); - IdentityType identityType = IdentityType.fromId(requireField(IDD_IDENTITY_TYPE_FIELD, root).asString()); - var clusterTypeField = root.field(IDD_CLUSTER_TYPE_FIELD); - var clusterType = clusterTypeField.valid() ? ClusterType.from(clusterTypeField.asString()) : null; - - - return new SignedIdentityDocument(signature, (int)signingKeyVersion, providerUniqueId, athenzService, (int)documentVersion, - configserverHostname, instanceHostname, createdAt, ips, identityType, clusterType); - } - - private static Instant getJsr310Instant(double v) { - try { - return objectMapper.readValue(Double.toString(v), Instant.class); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private static Cursor requireField(String fieldName, Cursor root) { - var field = root.field(fieldName); - if (!field.valid()) throw new IllegalArgumentException("Missing required field '" + fieldName + "'"); - return field; - } - -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AutoGeneratedKeyProvider.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AutoGeneratedKeyProvider.java deleted file mode 100644 index 67e5caa0c18..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AutoGeneratedKeyProvider.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyUtils; - -import java.security.KeyPair; -import java.security.KeyPairGenerator; -import java.security.NoSuchAlgorithmException; -import java.security.PrivateKey; -import java.security.PublicKey; - -/** - * @author bjorncs - */ -public class AutoGeneratedKeyProvider implements KeyProvider { - - private final KeyPair keyPair; - - public AutoGeneratedKeyProvider() { - keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); - } - - @Override - public PrivateKey getPrivateKey(int version) { - return keyPair.getPrivate(); - } - - @Override - public PublicKey getPublicKey(int version) { - return keyPair.getPublic(); - } - - public KeyPair getKeyPair() { - return keyPair; - } -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGeneratorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGeneratorTest.java deleted file mode 100644 index 9205baff0fc..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGeneratorTest.java +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.ClusterMembership; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; -import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import com.yahoo.vespa.athenz.identityprovider.client.IdentityDocumentSigner; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.Allocation; -import com.yahoo.vespa.hosted.provision.node.Generation; -import com.yahoo.vespa.hosted.provision.node.IP; -import com.yahoo.vespa.hosted.provision.node.Nodes; -import com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors; -import org.junit.jupiter.api.Test; - -import java.util.Optional; -import java.util.Set; - -import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.TestUtils.getAthenzProviderConfig; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author valerijf - */ -public class IdentityDocumentGeneratorTest { - - private static final Zone ZONE = new Zone(SystemName.cd, Environment.dev, RegionName.from("us-north-1")); - - @Test - void generates_valid_identity_document() { - String parentHostname = "docker-host"; - String containerHostname = "docker-container"; - - ApplicationId appid = ApplicationId.from( - TenantName.from("tenant"), ApplicationName.from("application"), InstanceName.from("default")); - Allocation allocation = new Allocation(appid, - ClusterMembership.from("container/default/0/0", Version.fromString("1.2.3"), Optional.empty()), - new NodeResources(1, 1, 1, 1), - Generation.initial(), - false); - Node parentNode = Node.create("ostkid", - IP.Config.ofEmptyPool(Set.of("127.0.0.1")), - parentHostname, - new MockNodeFlavors().getFlavorOrThrow("default"), - NodeType.host).build(); - Node containerNode = Node.reserve(Set.of("::1"), - containerHostname, - parentHostname, - new MockNodeFlavors().getFlavorOrThrow("default").resources(), - NodeType.tenant) - .allocation(allocation).build(); - NodeRepository nodeRepository = mock(NodeRepository.class); - Nodes nodes = mock(Nodes.class); - when(nodeRepository.nodes()).thenReturn(nodes); - - when(nodes.node(eq(parentHostname))).thenReturn(Optional.of(parentNode)); - when(nodes.node(eq(containerHostname))).thenReturn(Optional.of(containerNode)); - AutoGeneratedKeyProvider keyProvider = new AutoGeneratedKeyProvider(); - - String dnsSuffix = "vespa.dns.suffix"; - AthenzProviderServiceConfig config = getAthenzProviderConfig("domain", "service", dnsSuffix); - IdentityDocumentGenerator identityDocumentGenerator = - new IdentityDocumentGenerator(config, nodeRepository, ZONE, keyProvider); - SignedIdentityDocument signedIdentityDocument = identityDocumentGenerator.generateSignedIdentityDocument(containerHostname, IdentityType.TENANT); - - // Verify attributes - assertEquals(containerHostname, signedIdentityDocument.instanceHostname()); - - String environment = "dev"; - String region = "us-north-1"; - - VespaUniqueInstanceId expectedProviderUniqueId = - new VespaUniqueInstanceId(0, "default", "default", "application", "tenant", region, environment, IdentityType.TENANT); - assertEquals(expectedProviderUniqueId, signedIdentityDocument.providerUniqueId()); - - // Validate that container ips are present - assertTrue(signedIdentityDocument.ipAddresses().contains("::1")); - - IdentityDocumentSigner signer = new IdentityDocumentSigner(); - - // Validate signature - assertTrue(signer.hasValidSignature(signedIdentityDocument, keyProvider.getPublicKey(0))); - } -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidatorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidatorTest.java deleted file mode 100644 index b4b817da2f0..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidatorTest.java +++ /dev/null @@ -1,296 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.yahoo.component.Version; -import com.yahoo.config.model.api.ApplicationInfo; -import com.yahoo.config.model.api.HostInfo; -import com.yahoo.config.model.api.Model; -import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.config.model.api.SuperModel; -import com.yahoo.config.model.api.SuperModelProvider; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterMembership; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.identityprovider.api.ClusterType; -import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; -import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import com.yahoo.vespa.athenz.identityprovider.client.IdentityDocumentSigner; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceValidator.ValidationException; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeList; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.IP; -import com.yahoo.vespa.hosted.provision.node.Nodes; -import com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors; -import org.junit.jupiter.api.Test; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; - -import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceValidator.SERVICE_PROPERTIES_DOMAIN_KEY; -import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceValidator.SERVICE_PROPERTIES_SERVICE_KEY; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author valerijf - * @author bjorncs - * @author mortent - */ -public class InstanceValidatorTest { - - private final ApplicationId applicationId = ApplicationId.from("tenant", "application", "instance"); - private final String domain = "domain"; - private final String service = "service"; - - private final AthenzService vespaTenantDomain = new AthenzService("vespa.vespa.tenant"); - private final AutoGeneratedKeyProvider keyProvider = new AutoGeneratedKeyProvider(); - - @Test - void application_does_not_exist() { - SuperModelProvider superModelProvider = mockSuperModelProvider(); - InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, null, vespaTenantDomain); - assertFalse(instanceValidator.isValidInstance(createRegisterInstanceConfirmation(applicationId, domain, service))); - } - - @Test - void application_does_not_have_domain_set() { - SuperModelProvider superModelProvider = mockSuperModelProvider( - mockApplicationInfo(applicationId, 5, Collections.emptyList())); - InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, new IdentityDocumentSigner(), vespaTenantDomain); - - assertFalse(instanceValidator.isValidInstance(createRegisterInstanceConfirmation(applicationId, domain, service))); - } - - @Test - void application_has_wrong_domain() { - ServiceInfo serviceInfo = new ServiceInfo("serviceName", "type", Collections.emptyList(), - Collections.singletonMap(SERVICE_PROPERTIES_DOMAIN_KEY, "not-domain"), "confId", "hostName"); - - SuperModelProvider superModelProvider = mockSuperModelProvider( - mockApplicationInfo(applicationId, 5, Collections.singletonList(serviceInfo))); - InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, null, vespaTenantDomain); - - assertFalse(instanceValidator.isValidInstance(createRegisterInstanceConfirmation(applicationId, domain, service))); - } - - @Test - void application_has_same_domain_and_service() { - Map<String, String> properties = new HashMap<>(); - properties.put(SERVICE_PROPERTIES_DOMAIN_KEY, domain); - properties.put(SERVICE_PROPERTIES_SERVICE_KEY, service); - - ServiceInfo serviceInfo = new ServiceInfo("serviceName", "type", Collections.emptyList(), - properties, "confId", "hostName"); - - SuperModelProvider superModelProvider = mockSuperModelProvider( - mockApplicationInfo(applicationId, 5, Collections.singletonList(serviceInfo))); - IdentityDocumentSigner signer = mock(IdentityDocumentSigner.class); - when(signer.hasValidSignature(any(), any())).thenReturn(true); - InstanceValidator instanceValidator = new InstanceValidator(mock(KeyProvider.class), superModelProvider, mockNodeRepo(), signer, vespaTenantDomain); - - assertTrue(instanceValidator.isValidInstance(createRegisterInstanceConfirmation(applicationId, domain, service))); - } - - @Test - void rejects_invalid_provider_unique_id_in_csr() { - SuperModelProvider superModelProvider = mockSuperModelProvider(); - InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, null, vespaTenantDomain); - InstanceConfirmation instanceConfirmation = createRegisterInstanceConfirmation(applicationId, domain, service); - VespaUniqueInstanceId tamperedId = new VespaUniqueInstanceId(0, "default", "instance", "app", "tenant", "us-north-1", "dev", IdentityType.NODE); - instanceConfirmation.set("sanDNS", tamperedId.asDottedString() + ".instanceid.athenz.dev-us-north-1.vespa.yahoo.cloud"); - assertFalse(instanceValidator.isValidInstance(instanceConfirmation)); - } - - @Test - void rejects_unknown_ips_in_csr() { - NodeRepository nodeRepository = mockNodeRepo(); - InstanceValidator instanceValidator = new InstanceValidator(null, mockSuperModelProvider(), nodeRepository, null, vespaTenantDomain); - InstanceConfirmation instanceConfirmation = createRegisterInstanceConfirmation(applicationId, domain, service); - Set<String> nodeIp = nodeRepository.nodes().list().owner(applicationId).stream().findFirst() - .map(Node::ipConfig) - .map(IP.Config::primary) - .orElseThrow(() -> new RuntimeException("No ipaddress for mocked node")); - - List<String> ips = new ArrayList<>(nodeIp); - ips.add("::ff"); - instanceConfirmation.set("sanIP", String.join(",", ips)); - assertFalse(instanceValidator.isValidInstance(instanceConfirmation)); - } - - @Test - void rejects_invalid_cluster_type_in_csr() { - var props = Map.of(SERVICE_PROPERTIES_DOMAIN_KEY, domain, SERVICE_PROPERTIES_SERVICE_KEY, service); - var info = new ServiceInfo("serviceName", "type", List.of(), props, "confId", "hostName"); - var provider = mockSuperModelProvider(mockApplicationInfo(applicationId, 5, List.of(info))); - var instanceValidator = new InstanceValidator(keyProvider, provider, mockNodeRepo(), new IdentityDocumentSigner(), vespaTenantDomain); - var instanceConfirmation = createRegisterInstanceConfirmation(applicationId, domain, service); - instanceConfirmation.set("sanURI", "vespa://cluster-type/content"); - var exception = assertThrows(ValidationException.class, () -> instanceValidator.validateInstance(instanceConfirmation)); - var expectedMsg = "Illegal SAN URIs: expected '[vespa://cluster-type/container]' found '[vespa://cluster-type/content]'"; - assertEquals(expectedMsg, exception.getMessage()); - } - - @Test - void accepts_valid_refresh_requests() { - NodeRepository nodeRepository = mock(NodeRepository.class); - Nodes nodes = mock(Nodes.class); - when(nodeRepository.nodes()).thenReturn(nodes); - InstanceValidator instanceValidator = new InstanceValidator(null, null, nodeRepository, new IdentityDocumentSigner(), vespaTenantDomain); - - List<Node> nodeList = createNodes(10); - Node node = nodeList.get(0); - nodeList = allocateNode(nodeList, node, applicationId); - when(nodes.list()).thenReturn(NodeList.copyOf(nodeList)); - String nodeIp = node.ipConfig().primary().stream().findAny().orElseThrow(() -> new RuntimeException("No ipaddress for mocked node")); - InstanceConfirmation instanceConfirmation = createRefreshInstanceConfirmation(applicationId, domain, service, List.of(nodeIp)); - - assertTrue(instanceValidator.isValidRefresh(instanceConfirmation)); - } - - @Test - void rejects_refresh_on_ip_mismatch() { - NodeRepository nodeRepository = mockNodeRepo(); - InstanceValidator instanceValidator = new InstanceValidator(null, null, nodeRepository, new IdentityDocumentSigner(), vespaTenantDomain); - - Set<String> nodeIp = nodeRepository.nodes().list().owner(applicationId).stream().findFirst() - .map(Node::ipConfig) - .map(IP.Config::primary) - .orElseThrow(() -> new RuntimeException("No ipaddress for mocked node")); - - List<String> ips = new ArrayList<>(nodeIp); - ips.add("::ff"); - // Add invalid ip to list of ip addresses - InstanceConfirmation instanceConfirmation = createRefreshInstanceConfirmation(applicationId, domain, service, ips); - - assertFalse(instanceValidator.isValidRefresh(instanceConfirmation)); - } - - @Test - void rejects_refresh_when_node_is_not_allocated() { - NodeRepository nodeRepository = mock(NodeRepository.class); - Nodes nodes = mock(Nodes.class); - when(nodeRepository.nodes()).thenReturn(nodes); - - InstanceValidator instanceValidator = new InstanceValidator(null, null, nodeRepository, new IdentityDocumentSigner(), vespaTenantDomain); - - List<Node> nodeList = createNodes(10); - - when(nodes.list()).thenReturn(NodeList.copyOf(nodeList)); - InstanceConfirmation instanceConfirmation = createRefreshInstanceConfirmation(applicationId, domain, service, List.of("::11")); - - assertFalse(instanceValidator.isValidRefresh(instanceConfirmation)); - - } - - private NodeRepository mockNodeRepo() { - NodeRepository nodeRepository = mock(NodeRepository.class); - Nodes nodes = mock(Nodes.class); - when(nodeRepository.nodes()).thenReturn(nodes); - List<Node> nodeList = createNodes(10); - Node node = nodeList.get(0); - nodeList = allocateNode(nodeList, node, applicationId); - when(nodes.list()).thenReturn(NodeList.copyOf(nodeList)); - return nodeRepository; - } - - private InstanceConfirmation createRegisterInstanceConfirmation( - ApplicationId applicationId, String domain, String service) { - VespaUniqueInstanceId vespaUniqueInstanceId = new VespaUniqueInstanceId(0, "default", applicationId.instance().value(), applicationId.application().value(), applicationId.tenant().value(), "us-north-1", "dev", IdentityType.NODE); - var domainService = new AthenzService(domain, service); - var clock = Instant.now(); - var clusterType = ClusterType.CONTAINER; - var signature = new IdentityDocumentSigner() - .generateSignature( - vespaUniqueInstanceId, domainService, "localhost", "localhost", clock, Set.of(), - IdentityType.NODE, keyProvider.getPrivateKey(0)); - SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument( - signature, 0, vespaUniqueInstanceId, domainService, 0, "localhost", "localhost", - clock, Collections.emptySet(), IdentityType.NODE, clusterType); - return createInstanceConfirmation(vespaUniqueInstanceId, domain, service, signedIdentityDocument); - } - - private InstanceConfirmation createRefreshInstanceConfirmation(ApplicationId applicationId, String domain, String service, List<String> ips) { - VespaUniqueInstanceId vespaUniqueInstanceId = new VespaUniqueInstanceId(0, "default", applicationId.instance().value(), applicationId.application().value(), applicationId.tenant().value(), "us-north-1", "dev", IdentityType.NODE); - InstanceConfirmation instanceConfirmation = createInstanceConfirmation(vespaUniqueInstanceId, domain, service, null); - instanceConfirmation.set("sanIP", String.join(",", ips)); - return instanceConfirmation; - } - - private InstanceConfirmation createInstanceConfirmation(VespaUniqueInstanceId vespaUniqueInstanceId, String domain, String service, SignedIdentityDocument identityDocument) { - InstanceConfirmation instanceConfirmation = new InstanceConfirmation( - "vespa.vespa.cd.provider_dev_us-north-1", - domain, - service, - Optional.ofNullable(identityDocument) - .map(EntityBindingsMapper::toSignedIdentityDocumentEntity) - .orElse(null)); - instanceConfirmation.set("sanDNS", vespaUniqueInstanceId.asDottedString() + ".instanceid.athenz.dev-us-north-1.vespa.yahoo.cloud"); - instanceConfirmation.set("sanURI", "vespa://cluster-type/container"); - return instanceConfirmation; - } - - private SuperModelProvider mockSuperModelProvider(ApplicationInfo... appInfos) { - SuperModel superModel = new SuperModel(Stream.of(appInfos) - .collect(Collectors.toMap( - ApplicationInfo::getApplicationId, - Function.identity() - ) - ), - true); - - SuperModelProvider superModelProvider = mock(SuperModelProvider.class); - when(superModelProvider.getSuperModel()).thenReturn(superModel); - return superModelProvider; - } - - private ApplicationInfo mockApplicationInfo(ApplicationId appId, int numHosts, List<ServiceInfo> serviceInfo) { - List<HostInfo> hosts = IntStream.range(0, numHosts) - .mapToObj(i -> new HostInfo("host-" + i + "." + appId.toShortString() + ".yahoo.com", serviceInfo)) - .toList(); - - Model model = mock(Model.class); - when(model.getHosts()).thenReturn(hosts); - - return new ApplicationInfo(appId, 0, model); - } - - private List<Node> createNodes(int num) { - MockNodeFlavors flavors = new MockNodeFlavors(); - List<Node> nodeList = new ArrayList<>(); - for (int i = 0; i < num; i++) { - Node node = Node.create("foo" + i, IP.Config.of(Set.of("::1" + i, "::2" + i, "::3" + i), Set.of()), - "foo" + i, flavors.getFlavorOrThrow("default"), NodeType.tenant).build(); - nodeList.add(node); - } - return nodeList; - } - - private List<Node> allocateNode(List<Node> nodeList, Node node, ApplicationId applicationId) { - nodeList.removeIf(n -> n.id().equals(node.id())); - nodeList.add(node.allocate(applicationId, - ClusterMembership.from("container/default/0/0", Version.fromString("6.123.4"), Optional.empty()), - new NodeResources(1, 1, 1, 1), - Instant.now())); - return nodeList; - } -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/TestUtils.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/TestUtils.java deleted file mode 100644 index 4110ad2bfa2..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/TestUtils.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; - -/** - * @author bjorncs - */ -public class TestUtils { - - public static AthenzProviderServiceConfig getAthenzProviderConfig(String domain, - String service, - String dnsSuffix) { - AthenzProviderServiceConfig.Builder zoneConfig = - new AthenzProviderServiceConfig.Builder() - .serviceName(service) - .secretVersion(0) - .domain(domain) - .certDnsSuffix(dnsSuffix) - .ztsUrl("localhost/zts") - .secretName("s3cr3t") - .caCertSecretName(domain + ".ca.cert"); - return new AthenzProviderServiceConfig( - zoneConfig.athenzCaTrustStore("/dummy/path/to/athenz-ca.jks")); - } - -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificateTester.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificateTester.java deleted file mode 100644 index 4012776949e..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificateTester.java +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca; - -import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.Pkcs10Csr; -import com.yahoo.security.Pkcs10CsrBuilder; -import com.yahoo.security.SignatureAlgorithm; -import com.yahoo.security.SubjectAlternativeName; -import com.yahoo.security.X509CertificateBuilder; - -import javax.security.auth.x500.X500Principal; -import java.math.BigInteger; -import java.security.KeyPair; -import java.security.cert.X509Certificate; -import java.time.Duration; -import java.time.Instant; -import java.util.List; - -import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; - -/** - * Helper class for creating certificates, CSRs etc. for testing purposes. - * - * @author mpolden - */ -public class CertificateTester { - - private CertificateTester() {} - - public static X509Certificate createCertificate() { - var keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); - return createCertificate("subject", keyPair); - } - - public static X509Certificate createCertificate(String cn, KeyPair keyPair) { - var subject = new X500Principal("CN=" + cn); - return X509CertificateBuilder.fromKeypair(keyPair, - subject, - Instant.EPOCH, - Instant.EPOCH.plus(Duration.ofMinutes(1)), - SHA256_WITH_ECDSA, - BigInteger.ONE) - .build(); - } - - public static Pkcs10Csr createCsr() { - return createCsr(List.of(), List.of()); - } - - public static Pkcs10Csr createCsr(String dnsName) { - return createCsr(List.of(dnsName), List.of()); - } - - public static Pkcs10Csr createCsr(List<String> dnsNames) { - return createCsr(dnsNames, List.of()); - } - - public static Pkcs10Csr createCsr(String cn, List<String> dnsNames) { - return createCsr(cn, dnsNames, List.of()); - } - - public static Pkcs10Csr createCsr(List<String> dnsNames, List<String> ipAddresses) { - return createCsr("subject", dnsNames, ipAddresses); - } - public static Pkcs10Csr createCsr(String cn, List<String> dnsNames, List<String> ipAddresses) { - X500Principal subject = new X500Principal("CN=" + cn); - KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); - var builder = Pkcs10CsrBuilder.fromKeypair(subject, keyPair, SignatureAlgorithm.SHA512_WITH_ECDSA); - for (var dnsName : dnsNames) { - builder = builder.addSubjectAlternativeName(SubjectAlternativeName.Type.DNS, dnsName); - } - for (var ipAddress : ipAddresses) { - builder = builder.addSubjectAlternativeName(SubjectAlternativeName.Type.IP, ipAddress); - } - return builder.build(); - } - -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificatesTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificatesTest.java deleted file mode 100644 index dd3ddeeb804..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificatesTest.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca; - -import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.SubjectAlternativeName; -import com.yahoo.test.ManualClock; -import org.junit.jupiter.api.Test; - -import java.security.KeyPair; -import java.security.cert.X509Certificate; -import java.time.Duration; -import java.util.List; - -import static java.time.temporal.ChronoUnit.SECONDS; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - -/** - * @author mpolden - */ -public class CertificatesTest { - - private final KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); - private final X509Certificate caCertificate = CertificateTester.createCertificate("CA", keyPair); - - @Test - void expiry() { - var clock = new ManualClock(); - var certificates = new Certificates(clock); - var csr = CertificateTester.createCsr(); - var certificate = certificates.create(csr, caCertificate, keyPair.getPrivate()); - var now = clock.instant(); - - assertEquals(now.minus(Duration.ofHours(1)).truncatedTo(SECONDS), certificate.getNotBefore().toInstant()); - assertEquals(now.plus(Duration.ofDays(30)).truncatedTo(SECONDS), certificate.getNotAfter().toInstant()); - } - - @Test - void add_san_from_csr() throws Exception { - var certificates = new Certificates(new ManualClock()); - var dnsName = "host.example.com"; - var ip = "192.0.2.42"; - var csr = CertificateTester.createCsr(List.of(dnsName), List.of(ip)); - var certificate = certificates.create(csr, caCertificate, keyPair.getPrivate()); - - assertNotNull(certificate.getSubjectAlternativeNames()); - assertEquals(2, certificate.getSubjectAlternativeNames().size()); - - var subjectAlternativeNames = List.copyOf(certificate.getSubjectAlternativeNames()); - assertEquals(List.of(SubjectAlternativeName.Type.DNS.getTag(), dnsName), - subjectAlternativeNames.get(0)); - assertEquals(List.of(SubjectAlternativeName.Type.IP.getTag(), ip), - subjectAlternativeNames.get(1)); - } - - @Test - void parse_instance_id() { - var instanceId = "1.cluster1.default.app1.tenant1.us-north-1.prod.node"; - var instanceIdWithSuffix = instanceId + ".instanceid.athenz.dev-us-north-1.vespa.aws.oath.cloud"; - var csr = CertificateTester.createCsr(List.of("foo", "bar", instanceIdWithSuffix)); - assertEquals(instanceId, Certificates.instanceIdFrom(csr)); - } - -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java deleted file mode 100644 index bf2115e8759..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.restapi; - -import com.yahoo.application.container.handler.Request; -import com.yahoo.jdisc.http.server.jetty.RequestUtils; -import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.Pkcs10Csr; -import com.yahoo.security.Pkcs10CsrUtils; -import com.yahoo.security.X509CertificateUtils; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.text.StringUtilities; -import com.yahoo.vespa.athenz.api.AthenzPrincipal; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.client.ErrorHandler; -import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; -import com.yahoo.vespa.hosted.ca.CertificateTester; -import org.apache.http.client.ResponseHandler; -import org.apache.http.client.methods.HttpUriRequest; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import javax.net.ssl.SSLContext; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.security.Principal; -import java.security.cert.X509Certificate; -import java.util.List; -import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - -/** - * @author mpolden - */ -public class CertificateAuthorityApiTest extends ContainerTester { - - private static final String INSTANCE_ID = "1.cluster1.default.app1.tenant1.us-north-1.prod.node"; - private static final String INSTANCE_ID_WITH_SUFFIX = INSTANCE_ID + ".instanceid.athenz.dev-us-north-1.vespa.aws.oath.cloud"; - private static final String INVALID_INSTANCE_ID = "1.cluster1.default.otherapp.othertenant.us-north-1.prod.node"; - private static final String INVALID_INSTANCE_ID_WITH_SUFFIX = INVALID_INSTANCE_ID + ".instanceid.athenz.dev-us-north-1.vespa.aws.oath.cloud"; - - private static final String CONTAINER_IDENTITY = "vespa.external.tenant"; - private static final String HOST_IDENTITY = "vespa.external.tenant-host"; - - @BeforeEach - public void before() { - setCaCertificateAndKey(); - } - - @Test - void register_instance() throws Exception { - // POST instance registration - var csr = CertificateTester.createCsr(List.of("node1.example.com", INSTANCE_ID_WITH_SUFFIX)); - assertIdentityResponse(new Request("http://localhost:12345/ca/v1/instance/", - instanceRegistrationJson(csr), - Request.Method.POST)); - - // POST instance registration with ZTS client - var ztsClient = new TestZtsClient(new AthenzPrincipal(new AthenzService(HOST_IDENTITY)), null, URI.create("http://localhost:12345/ca/v1/"), SSLContext.getDefault()); - var instanceIdentity = ztsClient.registerInstance(new AthenzService("vespa.external", "provider_prod_us-north-1"), - new AthenzService(CONTAINER_IDENTITY), - getAttestationData(), - csr); - assertEquals("CN=Vespa CA", instanceIdentity.certificate().getIssuerX500Principal().getName()); - } - - private X509Certificate registerInstance() throws Exception { - // POST instance registration - var csr = CertificateTester.createCsr(CONTAINER_IDENTITY, List.of("node1.example.com", INSTANCE_ID_WITH_SUFFIX)); - assertIdentityResponse(new Request("http://localhost:12345/ca/v1/instance/", - instanceRegistrationJson(csr), - Request.Method.POST)); - - // POST instance registration with ZTS client - var ztsClient = new TestZtsClient(new AthenzPrincipal(new AthenzService(HOST_IDENTITY)), null, URI.create("http://localhost:12345/ca/v1/"), SSLContext.getDefault()); - var instanceIdentity = ztsClient.registerInstance(new AthenzService("vespa.external", "provider_prod_us-north-1"), - new AthenzService(CONTAINER_IDENTITY), - getAttestationData(), - csr); - return instanceIdentity.certificate(); - } - - @Test - void refresh_instance() throws Exception { - // Register instance to get cert - var certificate = registerInstance(); - - // POST instance refresh - var principal = new AthenzPrincipal(new AthenzService(CONTAINER_IDENTITY)); - var csr = CertificateTester.createCsr(principal.getIdentity().getFullName(), List.of("node1.example.com", INSTANCE_ID_WITH_SUFFIX)); - var request = new Request("http://localhost:12345/ca/v1/instance/vespa.external.provider_prod_us-north-1/vespa.external/tenant/" + INSTANCE_ID, - instanceRefreshJson(csr), - Request.Method.POST, - principal); - request.getAttributes().put(RequestUtils.JDISC_REQUEST_X509CERT, new X509Certificate[]{certificate}); - assertIdentityResponse(request); - - // POST instance refresh with ZTS client - var ztsClient = new TestZtsClient(principal, certificate, URI.create("http://localhost:12345/ca/v1/"), SSLContext.getDefault()); - var instanceIdentity = ztsClient.refreshInstance(new AthenzService("vespa.external", "provider_prod_us-north-1"), - new AthenzService(CONTAINER_IDENTITY), - INSTANCE_ID, - csr); - assertEquals("CN=Vespa CA", instanceIdentity.certificate().getIssuerX500Principal().getName()); - } - - @Test - void invalid_requests() throws Exception { - // POST instance registration with missing fields - assertResponse(400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"POST http://localhost:12345/ca/v1/instance/ failed: Missing required field 'provider'\"}", - new Request("http://localhost:12345/ca/v1/instance/", - new byte[0], - Request.Method.POST)); - - // POST instance registration without DNS name in CSR - var csr = CertificateTester.createCsr(); - var request = new Request("http://localhost:12345/ca/v1/instance/", - instanceRegistrationJson(csr), - Request.Method.POST); - assertResponse(400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"POST http://localhost:12345/ca/v1/instance/ failed: No instance ID found in CSR\"}", request); - - // POST instance refresh with missing field - assertResponse(400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"POST http://localhost:12345/ca/v1/instance/vespa.external.provider_prod_us-north-1/vespa.external/tenant/1.cluster1.default.app1.tenant1.us-north-1.prod.node failed: Missing required field 'csr'\"}", - new Request("http://localhost:12345/ca/v1/instance/vespa.external.provider_prod_us-north-1/vespa.external/tenant/" + INSTANCE_ID, - new byte[0], - Request.Method.POST)); - - // POST instance refresh where instanceId does not match CSR dnsName - var principal = new AthenzPrincipal(new AthenzService(CONTAINER_IDENTITY)); - var cert = CertificateTester.createCertificate(CONTAINER_IDENTITY, KeyUtils.generateKeypair(KeyAlgorithm.EC)); - csr = CertificateTester.createCsr(principal.getIdentity().getFullName(), List.of("node1.example.com", INSTANCE_ID_WITH_SUFFIX)); - request = new Request("http://localhost:12345/ca/v1/instance/vespa.external.provider_prod_us-north-1/vespa.external/tenant/foobar", - instanceRefreshJson(csr), - Request.Method.POST, - principal); - request.getAttributes().put(RequestUtils.JDISC_REQUEST_X509CERT, new X509Certificate[]{cert}); - assertResponse( - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"POST http://localhost:12345/ca/v1/instance/vespa.external.provider_prod_us-north-1/vespa.external/tenant/foobar failed: Mismatch between instance ID in URL path and instance ID in CSR [instanceId=foobar,instanceIdFromCsr=1.cluster1.default.app1.tenant1.us-north-1.prod.node]\"}", - request); - - // POST instance refresh using zts client where client cert does not contain instanceid - var certificate = registerInstance(); - var ztsClient = new TestZtsClient(principal, certificate, URI.create("http://localhost:12345/ca/v1/"), SSLContext.getDefault()); - try { - var invalidCsr = CertificateTester.createCsr(principal.getIdentity().getFullName(), List.of("node1.example.com", INVALID_INSTANCE_ID_WITH_SUFFIX)); - var instanceIdentity = ztsClient.refreshInstance(new AthenzService("vespa.external", "provider_prod_us-north-1"), - new AthenzService(CONTAINER_IDENTITY), - INSTANCE_ID, - invalidCsr); - fail("Refresh instance should have failed"); - } catch (Exception e) { - String expectedMessage = "Received error from ZTS: code=0, message=\"POST http://localhost:12345/ca/v1/instance/vespa.external.provider_prod_us-north-1/vespa.external/tenant/1.cluster1.default.app1.tenant1.us-north-1.prod.node failed: Mismatch between instance ID in URL path and instance ID in CSR [instanceId=1.cluster1.default.app1.tenant1.us-north-1.prod.node,instanceIdFromCsr=1.cluster1.default.otherapp.othertenant.us-north-1.prod.node]\""; - assertEquals(expectedMessage, e.getMessage()); - } - } - - private void setCaCertificateAndKey() { - var keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); - var caCertificatePem = X509CertificateUtils.toPem(CertificateTester.createCertificate("Vespa CA", keyPair)); - var privateKeyPem = KeyUtils.toPem(keyPair.getPrivate()); - secretStore().setSecret("vespa.external.ca.cert", caCertificatePem) - .setSecret("secretname", privateKeyPem); - } - - private void assertIdentityResponse(Request request) { - assertResponse(200, (body) -> { - var slime = SlimeUtils.jsonToSlime(body); - var root = slime.get(); - assertEquals("vespa.external.provider_prod_us-north-1", root.field("provider").asString()); - assertEquals("tenant", root.field("service").asString()); - assertEquals(INSTANCE_ID, root.field("instanceId").asString()); - var pemEncodedCertificate = root.field("x509Certificate").asString(); - assertTrue(pemEncodedCertificate.startsWith("-----BEGIN CERTIFICATE-----") && - pemEncodedCertificate.endsWith("-----END CERTIFICATE-----\n"), - "Response contains PEM certificate"); - }, request); - } - - private static byte[] instanceRefreshJson(Pkcs10Csr csr) { - var csrPem = Pkcs10CsrUtils.toPem(csr); - var json = "{\"csr\": \"" + csrPem + "\"}"; - return json.getBytes(StandardCharsets.UTF_8); - } - - private static byte[] instanceRegistrationJson(Pkcs10Csr csr) { - var csrPem = Pkcs10CsrUtils.toPem(csr); - var json = "{\n" + - " \"provider\": \"vespa.external.provider_prod_us-north-1\",\n" + - " \"domain\": \"vespa.external\",\n" + - " \"service\": \"tenant\",\n" + - " \"attestationData\": \""+getAttestationData()+"\",\n" + - " \"csr\": \"" + csrPem + "\"\n" + - "}"; - return json.getBytes(StandardCharsets.UTF_8); - } - - private static String getAttestationData () { - var json = "{\n" + - " \"signature\": \"SIGNATURE\",\n" + - " \"signing-key-version\": 0,\n" + - " \"provider-unique-id\": \"0.default.default.application.tenant.us-north-1.dev.tenant\",\n" + - " \"provider-service\": \"domain.service\",\n" + - " \"document-version\": 1,\n" + - " \"configserver-hostname\": \"localhost\",\n" + - " \"instance-hostname\": \"docker-container\",\n" + - " \"created-at\": 1572000079.00000,\n" + - " \"ip-addresses\": [\n" + - " \"::1\"\n" + - " ],\n" + - " \"identity-type\": \"tenant\"\n" + - "}"; - return StringUtilities.escape(json); - } - - /* - Zts client that adds principal as header (since setting up ssl in test is cumbersome) - */ - private static class TestZtsClient extends DefaultZtsClient { - - private final Principal principal; - private final X509Certificate certificate; - - public TestZtsClient(Principal principal, X509Certificate certificate, URI ztsUrl, SSLContext sslContext) { - super(ztsUrl, () -> sslContext, null, ErrorHandler.empty()); - this.principal = principal; - this.certificate = certificate; - } - - @Override - protected <T> T execute(HttpUriRequest request, ResponseHandler<T> responseHandler) { - request.addHeader("PRINCIPAL", principal.getName()); - Optional.ofNullable(certificate).ifPresent(cert -> { - var pem = X509CertificateUtils.toPem(certificate); - request.addHeader("CERTIFICATE", StringUtilities.escape(pem)); - }); - return super.execute(request, responseHandler); - } - } -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java deleted file mode 100644 index 8112f5779e5..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.restapi; - -import com.yahoo.application.Networking; -import com.yahoo.application.container.JDisc; -import com.yahoo.application.container.handler.Request; -import com.yahoo.vespa.hosted.ca.restapi.mock.SecretStoreMock; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; - -import java.io.UncheckedIOException; -import java.nio.charset.CharacterCodingException; -import java.util.function.Consumer; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * The superclass of REST API tests which require a functional container instance. - * - * @author mpolden - */ -public class ContainerTester { - - private JDisc container; - - @BeforeEach - public void startContainer() { - container = JDisc.fromServicesXml(servicesXml(), Networking.enable); - } - - @AfterEach - public void stopContainer() { - container.close(); - } - - public SecretStoreMock secretStore() { - return (SecretStoreMock) container.components().getComponent(SecretStoreMock.class.getName()); - } - - public void assertResponse(int expectedStatus, String expectedBody, Request request) { - assertResponse(expectedStatus, (body) -> assertEquals(expectedBody, body), request); - } - - public void assertResponse(int expectedStatus, Consumer<String> bodyAsserter, Request request) { - var response = container.handleRequest(request); - try { - bodyAsserter.accept(response.getBodyAsString()); - } catch (CharacterCodingException e) { - throw new UncheckedIOException(e); - } - assertEquals(expectedStatus, response.getStatus()); - assertEquals("application/json; charset=UTF-8", response.getHeaders().getFirst("Content-Type")); - } - - private static String servicesXml() { - return "<container version='1.0'>\n" + - " <accesslog type=\"disabled\"/>\n" + - " <config name=\"container.handler.threadpool\">\n" + - " <maxthreads>10</maxthreads>\n" + - " </config>\n" + - " <config name='vespa.hosted.athenz.instanceproviderservice.config.athenz-provider-service'>\n" + - " <athenzCaTrustStore>/path/to/file</athenzCaTrustStore>\n" + - " <domain>vespa.external</domain>\n" + - " <serviceName>servicename</serviceName>\n" + - " <secretName>secretname</secretName>\n" + - " <secretVersion>0</secretVersion>\n" + - " <caCertSecretName>vespa.external.ca.cert</caCertSecretName>\n" + - " <certDnsSuffix>suffix</certDnsSuffix>\n" + - " <ztsUrl>https://localhost:123/</ztsUrl>\n" + - " </config>\n" + - " <component id='com.yahoo.vespa.hosted.ca.restapi.mock.SecretStoreMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.ca.restapi.mock.InstanceValidatorMock'/>\n" + - " <handler id='com.yahoo.vespa.hosted.ca.restapi.CertificateAuthorityApiHandler'>\n" + - " <binding>http://*/ca/v1/*</binding>\n" + - " </handler>\n" + - " <http>\n" + - " <server id='default' port='12345'/>\n" + - " <filtering>\n" + - " <request-chain id=\"my-default-chain\">\n" + - " <filter id='com.yahoo.vespa.hosted.ca.restapi.mock.PrincipalFromHeaderFilter' />\n" + - " <binding>http://*/*</binding>\n" + - " </request-chain>\n" + - " </filtering>\n" + - " </http>\n" + - "</container>"; - } - -}
\ No newline at end of file diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java deleted file mode 100644 index ca624918beb..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.restapi; - -import com.yahoo.security.Pkcs10CsrUtils; -import com.yahoo.security.X509CertificateUtils; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.text.StringUtilities; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.identityprovider.api.ClusterType; -import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; -import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import com.yahoo.vespa.hosted.ca.CertificateTester; -import com.yahoo.vespa.hosted.ca.instance.InstanceIdentity; -import com.yahoo.vespa.hosted.ca.instance.InstanceRefresh; -import com.yahoo.vespa.hosted.ca.instance.InstanceRegistration; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.Collections; -import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author mpolden - */ -public class InstanceSerializerTest { - - @Test - void deserialize_instance_registration() { - var csr = CertificateTester.createCsr(); - var csrPem = Pkcs10CsrUtils.toPem(csr); - SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument( - "signature", - 0, - new VespaUniqueInstanceId(0, "cluster", "instance", "application", "tenant", "region", "prod", IdentityType.NODE), - new AthenzService("domain", "service"), - 0, - "configserverhostname", - "instancehostname", - Instant.now().truncatedTo(ChronoUnit.MICROS), // Truncate to the precision given from EntityBindingsMapper.toAttestationData() - Collections.emptySet(), - IdentityType.NODE, - ClusterType.CONTAINER); - - var json = String.format("{\n" + - " \"provider\": \"provider_prod_us-north-1\",\n" + - " \"domain\": \"vespa.external\",\n" + - " \"service\": \"tenant\",\n" + - " \"attestationData\":\"%s\",\n" + - " \"csr\": \"" + csrPem + "\"\n" + - "}", StringUtilities.escape(EntityBindingsMapper.toAttestationData(signedIdentityDocument))); - var instanceRegistration = new InstanceRegistration("provider_prod_us-north-1", "vespa.external", - "tenant", signedIdentityDocument, - csr); - var deserialized = InstanceSerializer.registrationFromSlime(SlimeUtils.jsonToSlime(json)); - assertEquals(instanceRegistration, deserialized); - } - - @Test - void serialize_instance_identity() { - var certificate = CertificateTester.createCertificate(); - var pem = X509CertificateUtils.toPem(certificate); - var identity = new InstanceIdentity("provider_prod_us-north-1", "tenant", "node1.example.com", - Optional.of(certificate)); - var json = "{" + - "\"provider\":\"provider_prod_us-north-1\"," + - "\"service\":\"tenant\"," + - "\"instanceId\":\"node1.example.com\"," + - "\"x509Certificate\":\"" + pem.replace("\n", "\\n") + "\"" + - "}"; - assertEquals(json, asJsonString(InstanceSerializer.identityToSlime(identity))); - } - - @Test - void serialize_instance_refresh() { - var csr = CertificateTester.createCsr(); - var csrPem = Pkcs10CsrUtils.toPem(csr); - var json = "{\"csr\": \"" + csrPem + "\"}"; - var instanceRefresh = new InstanceRefresh(csr); - var deserialized = InstanceSerializer.refreshFromSlime(SlimeUtils.jsonToSlime(json)); - assertEquals(instanceRefresh, deserialized); - } - - private static String asJsonString(Slime slime) { - try { - return new String(SlimeUtils.toJsonBytes(slime), StandardCharsets.UTF_8); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/InstanceValidatorMock.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/InstanceValidatorMock.java deleted file mode 100644 index 4151c1f15d7..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/InstanceValidatorMock.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.restapi.mock; - -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceConfirmation; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceValidator; - -/** - * @author mortent - */ -public class InstanceValidatorMock extends InstanceValidator { - - public InstanceValidatorMock() { - super(null, null, null, null, null); - } - - @Override - public boolean isValidInstance(InstanceConfirmation instanceConfirmation) { - return instanceConfirmation.attributes.get(SAN_DNS_ATTRNAME) != null && - instanceConfirmation.attributes.get(SAN_IPS_ATTRNAME) != null; - } - - @Override - public boolean isValidRefresh(InstanceConfirmation confirmation) { - return confirmation.attributes.get(SAN_DNS_ATTRNAME) != null && - confirmation.attributes.get(SAN_IPS_ATTRNAME) != null; - } -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/PrincipalFromHeaderFilter.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/PrincipalFromHeaderFilter.java deleted file mode 100644 index df98ba75dd2..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/PrincipalFromHeaderFilter.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.restapi.mock; - -import com.yahoo.jdisc.handler.ResponseHandler; -import com.yahoo.jdisc.http.filter.DiscFilterRequest; -import com.yahoo.jdisc.http.filter.SecurityRequestFilter; -import com.yahoo.jdisc.http.server.jetty.RequestUtils; -import com.yahoo.security.X509CertificateUtils; -import com.yahoo.text.StringUtilities; -import com.yahoo.vespa.athenz.api.AthenzPrincipal; -import com.yahoo.vespa.athenz.api.AthenzService; - -import java.security.cert.X509Certificate; -import java.util.Optional; - -/** - * Read principal from http header - * - * @author mortent - */ -public class PrincipalFromHeaderFilter implements SecurityRequestFilter { - - @Override - public void filter(DiscFilterRequest request, ResponseHandler handler) { - String principal = request.getHeader("PRINCIPAL"); - request.setUserPrincipal(new AthenzPrincipal(new AthenzService(principal))); - - Optional<String> certificate = Optional.ofNullable(request.getHeader("CERTIFICATE")); - certificate.ifPresent(cert -> { - var x509cert = X509CertificateUtils.fromPem(StringUtilities.unescape(cert)); - request.setAttribute(RequestUtils.JDISC_REQUEST_X509CERT, new X509Certificate[]{x509cert}); - }); - } -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/SecretStoreMock.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/SecretStoreMock.java deleted file mode 100644 index 5a9f4fd0b76..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/SecretStoreMock.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.restapi.mock; - -import com.yahoo.component.AbstractComponent; -import com.yahoo.container.jdisc.secretstore.SecretStore; - -import java.util.HashMap; -import java.util.Map; - -/** - * @author mpolden - */ -public class SecretStoreMock extends AbstractComponent implements SecretStore { - - private final Map<String, String> secrets = new HashMap<>(); - - public SecretStoreMock setSecret(String key, String value) { - secrets.put(key, value); - return this; - } - - @Override - public String getSecret(String key) { - if (!secrets.containsKey(key)) throw new RuntimeException("No such key '" + key + "'"); - return secrets.get(key); - } - - @Override - public String getSecret(String key, int version) { - if (!secrets.containsKey(key)) throw new RuntimeException("No such key '" + key + "'"); - return secrets.get(key); - } - -} diff --git a/bootstrap.sh b/bootstrap.sh index e8730303ef7..0108999b1c6 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -42,7 +42,7 @@ echo "Using maven command: ${MAVEN_CMD}" echo "Using maven extra opts: ${MAVEN_EXTRA_OPTS}" mvn_install() { - ${MAVEN_CMD} --no-snapshot-updates -Dmaven.wagon.http.retryHandler.count=5 clean install ${MAVEN_EXTRA_OPTS} "$@" + ${MAVEN_CMD} --batch-mode --no-snapshot-updates -Dmaven.wagon.http.retryHandler.count=5 clean install ${MAVEN_EXTRA_OPTS} "$@" } # Generate vtag map diff --git a/client/go/go.mod b/client/go/go.mod index fc5ae544dac..d03f22a9e67 100644 --- a/client/go/go.mod +++ b/client/go/go.mod @@ -13,7 +13,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 github.com/zalando/go-keyring v0.1.1 - golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c + golang.org/x/sys v0.5.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/client/go/go.sum b/client/go/go.sum index fedf23f3e0b..084bde701ce 100644 --- a/client/go/go.sum +++ b/client/go/go.sum @@ -4,13 +4,10 @@ github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKY github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/danieljoos/wincred v1.1.0 h1:3RNcEpBg4IhIChZdFRSdlQt1QjCp1sMAPIrOnm7Yf8g= github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -22,11 +19,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/pkg/browser v0.0.0-20210706143420-7d21f8c997e2 h1:acNfDZXmm28D2Yg/c3ALnZStzNaZMSagpbr96vY6Zjc= @@ -39,26 +33,17 @@ github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/zalando/go-keyring v0.1.1 h1:w2V9lcx/Uj4l+dzAf1m9s+DJ1O8ROkEHnynonHjTcYE= github.com/zalando/go-keyring v0.1.1/go.mod h1:OIC+OZ28XbmwFxU/Rp9V7eKzZjamBJwRzC8UFJH9+L8= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/client/go/internal/cli/cmd/root.go b/client/go/internal/cli/cmd/root.go index faba6bbbfd4..70e0afbcd32 100644 --- a/client/go/internal/cli/cmd/root.go +++ b/client/go/internal/cli/cmd/root.go @@ -250,6 +250,7 @@ func (c *CLI) configureCommands() { rootCmd.AddCommand(statusCmd) // status rootCmd.AddCommand(newTestCmd(c)) // test rootCmd.AddCommand(newVersionCmd(c)) // version + rootCmd.AddCommand(newVisitCmd(c)) // visit } func (c *CLI) printErr(err error, hints ...string) { @@ -263,6 +264,10 @@ func (c *CLI) printSuccess(msg ...interface{}) { fmt.Fprintln(c.Stdout, color.GreenString("Success:"), fmt.Sprint(msg...)) } +func (c *CLI) printDebug(msg ...interface{}) { + fmt.Fprintln(c.Stderr, color.CyanString("Debug:"), fmt.Sprint(msg...)) +} + func (c *CLI) printWarning(msg interface{}, hints ...string) { fmt.Fprintln(c.Stderr, color.YellowString("Warning:"), msg) for _, hint := range hints { diff --git a/client/go/internal/cli/cmd/visit.go b/client/go/internal/cli/cmd/visit.go new file mode 100644 index 00000000000..1022d74354d --- /dev/null +++ b/client/go/internal/cli/cmd/visit.go @@ -0,0 +1,348 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa visit command +// Author: arnej + +package cmd + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +type visitArgs struct { + contentCluster string + fieldSet string + selection string + makeFeed bool + jsonLines bool + pretty bool + debugMode bool + chunkCount int + cli *CLI +} + +func (v *visitArgs) writeBytes(b []byte) { + v.cli.Stdout.Write(b) +} + +func (v *visitArgs) writeString(s string) { + v.writeBytes([]byte(s)) +} + +func (v *visitArgs) debugPrint(s string) { + if v.debugMode { + v.cli.printDebug(s) + } +} + +func (v *visitArgs) dumpDocuments(documents []DocumentBlob) { + comma := false + pretty := false + if v.makeFeed { + comma = true + pretty = v.pretty + } else if !v.jsonLines { + return + } + for _, value := range documents { + if pretty { + var prettyJSON bytes.Buffer + parseError := json.Indent(&prettyJSON, value.blob, "", " ") + if parseError != nil { + v.writeBytes(value.blob) + } else { + v.writeBytes(prettyJSON.Bytes()) + } + } else { + v.writeBytes(value.blob) + } + if comma { + v.writeString(",\n") + } else { + v.writeString("\n") + } + } +} + +var totalDocCount int + +func newVisitCmd(cli *CLI) *cobra.Command { + var ( + vArgs visitArgs + ) + cmd := &cobra.Command{ + Use: "visit", + Short: "Visit and print all documents in a vespa cluster", + Long: `Run visiting to retrieve all documents. + +By default prints each document received on its own line (JSON-L format). +`, + Example: `$ vespa visit # get documents from any cluster +$ vespa visit --content-cluster search # get documents from cluster named "search" +`, + Args: cobra.MaximumNArgs(0), + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + vArgs.cli = cli + service, err := documentService(cli) + if err != nil { + return err + } + result := probeHandler(service, cli) + if result.Success { + result = visitClusters(&vArgs, service) + } + if !result.Success { + return fmt.Errorf("visit failed: %s", result.Message) + } + vArgs.debugPrint(fmt.Sprintf("sum of 'documentCount': %d", totalDocCount)) + return nil + }, + } + cmd.Flags().StringVar(&vArgs.contentCluster, "content-cluster", "*", `Which content cluster to visit documents from`) + cmd.Flags().StringVar(&vArgs.fieldSet, "field-set", "", `Which fieldset to ask for`) + cmd.Flags().StringVar(&vArgs.selection, "selection", "", `select subset of cluster`) + cmd.Flags().BoolVar(&vArgs.debugMode, "debug-mode", false, `print debugging output`) + cmd.Flags().BoolVar(&vArgs.jsonLines, "json-lines", true, `output documents as JSON lines`) + cmd.Flags().BoolVar(&vArgs.makeFeed, "make-feed", false, `output JSON array suitable for vespa-feeder`) + cmd.Flags().BoolVar(&vArgs.pretty, "pretty-json", false, `format pretty JSON`) + cmd.Flags().IntVar(&vArgs.chunkCount, "chunk-count", 1000, `chunk by count`) + return cmd +} + +type HandlersInfo struct { + Handlers []struct { + HandlerId string `json:"id"` + HandlerClass string `json:"class"` + HandlerBundle string `json:"bundle"` + ServerBindings []string `json:"serverBindings"` + } `json:"handlers"` +} + +func parseHandlersOutput(r io.Reader) (*HandlersInfo, error) { + var handlersInfo HandlersInfo + codec := json.NewDecoder(r) + err := codec.Decode(&handlersInfo) + return &handlersInfo, err +} + +func probeHandler(service *vespa.Service, cli *CLI) (res util.OperationResult) { + urlPath := service.BaseURL + "/" + url, urlParseError := url.Parse(urlPath) + if urlParseError != nil { + return util.Failure("Invalid request path: '" + urlPath + "': " + urlParseError.Error()) + } + request := &http.Request{ + URL: url, + Method: "GET", + } + timeout := time.Duration(90) * time.Second + response, err := service.Do(request, timeout) + if err != nil { + return util.Failure("Request failed: " + err.Error()) + } + defer response.Body.Close() + if response.StatusCode == 200 { + handlersInfo, err := parseHandlersOutput(response.Body) + if err != nil || len(handlersInfo.Handlers) == 0 { + cli.printWarning("Could not parse JSON response from"+urlPath, err.Error()) + return util.Failure("Bad endpoint") + } + for _, h := range handlersInfo.Handlers { + if strings.HasSuffix(h.HandlerClass, "DocumentV1ApiHandler") { + for _, binding := range h.ServerBindings { + if strings.Contains(binding, "/document/v1/") { + return util.Success("handler OK") + } + } + w := fmt.Sprintf("expected /document/v1/ binding, but got: %v", h.ServerBindings) + cli.printWarning(w) + } + } + cli.printWarning("Missing /document/v1/ API; add <document-api /> to the container cluster delcaration in services.xml") + return util.Failure("Missing /document/v1 API") + } else { + return util.FailureWithPayload(service.Description()+" at "+request.URL.Host+": "+response.Status, util.ReaderToJSON(response.Body)) + } +} + +func visitClusters(vArgs *visitArgs, service *vespa.Service) (res util.OperationResult) { + clusters := []string{ + vArgs.contentCluster, + } + if vArgs.contentCluster == "*" { + clusters = probeVisit(vArgs, service) + } + if vArgs.makeFeed { + vArgs.writeString("[\n") + } + for _, c := range clusters { + vArgs.contentCluster = c + res = runVisit(vArgs, service) + if !res.Success { + return res + } + vArgs.debugPrint("Success: " + res.Message) + } + if vArgs.makeFeed { + vArgs.writeString("{}\n]\n") + } + return res +} + +func probeVisit(vArgs *visitArgs, service *vespa.Service) []string { + clusters := make([]string, 0, 3) + vvo, _ := runOneVisit(vArgs, service, "") + if vvo != nil { + msg := vvo.ErrorMsg + if strings.Contains(msg, "no content cluster '*'") { + for idx, value := range strings.Split(msg, ",") { + if idx > 0 { + parts := strings.Split(value, "'") + if len(parts) == 3 { + clusters = append(clusters, parts[1]) + } + } + } + } + } + return clusters +} + +func runVisit(vArgs *visitArgs, service *vespa.Service) (res util.OperationResult) { + vArgs.debugPrint(fmt.Sprintf("trying to visit: '%s'", vArgs.contentCluster)) + var totalDocuments int = 0 + var continuationToken string + for { + var vvo *VespaVisitOutput + vvo, res = runOneVisit(vArgs, service, continuationToken) + if !res.Success { + if vvo != nil && vvo.ErrorMsg != "" { + vArgs.cli.printWarning(vvo.ErrorMsg) + } + return res + } + vArgs.dumpDocuments(vvo.Documents) + vArgs.debugPrint(fmt.Sprintf("got %d documents", len(vvo.Documents))) + totalDocuments += len(vvo.Documents) + continuationToken = vvo.Continuation + if continuationToken == "" { + break + } + } + res.Message = fmt.Sprintf("%s [%d documents visited]", res.Message, totalDocuments) + return +} + +func quoteArgForUrl(arg string) string { + var buf strings.Builder + buf.Grow(len(arg)) + for _, r := range arg { + switch { + case 'a' <= r && r <= 'z': + buf.WriteRune(r) + case 'A' <= r && r <= 'Z': + buf.WriteRune(r) + case '0' <= r && r <= '9': + buf.WriteRune(r) + case r <= ' ' || r > '~': + buf.WriteRune('+') + default: + s := fmt.Sprintf("%s%02X", "%", r) + buf.WriteString(s) + } + } + return buf.String() +} + +func runOneVisit(vArgs *visitArgs, service *vespa.Service, contToken string) (*VespaVisitOutput, util.OperationResult) { + urlPath := service.BaseURL + "/document/v1/?cluster=" + quoteArgForUrl(vArgs.contentCluster) + if vArgs.fieldSet != "" { + urlPath = urlPath + "&fieldSet=" + quoteArgForUrl(vArgs.fieldSet) + } + if vArgs.selection != "" { + urlPath = urlPath + "&selection=" + quoteArgForUrl(vArgs.selection) + } + if contToken != "" { + urlPath = urlPath + "&continuation=" + contToken + } + if vArgs.chunkCount > 0 { + urlPath = urlPath + fmt.Sprintf("&wantedDocumentCount=%d", vArgs.chunkCount) + } + url, urlParseError := url.Parse(urlPath) + if urlParseError != nil { + return nil, util.Failure("Invalid request path: '" + urlPath + "': " + urlParseError.Error()) + } + request := &http.Request{ + URL: url, + Method: "GET", + } + timeout := time.Duration(900) * time.Second + response, err := service.Do(request, timeout) + if err != nil { + return nil, util.Failure("Request failed: " + err.Error()) + } + defer response.Body.Close() + vvo, err := parseVisitOutput(response.Body) + if response.StatusCode == 200 { + if err == nil { + totalDocCount += vvo.DocumentCount + if vvo.DocumentCount != len(vvo.Documents) { + vArgs.cli.printWarning(fmt.Sprintf("Inconsistent contents from: %v", url)) + vArgs.cli.printWarning(fmt.Sprintf("claimed count: %d", vvo.DocumentCount)) + vArgs.cli.printWarning(fmt.Sprintf("document blobs: %d", len(vvo.Documents))) + return nil, util.Failure("Inconsistent contents from document API") + } + return vvo, util.Success("visited " + vArgs.contentCluster) + } else { + return nil, util.Failure("error reading response: " + err.Error()) + } + } else if response.StatusCode/100 == 4 { + return vvo, util.FailureWithPayload("Invalid document operation: "+response.Status, util.ReaderToJSON(response.Body)) + } else { + return vvo, util.FailureWithPayload(service.Description()+" at "+request.URL.Host+": "+response.Status, util.ReaderToJSON(response.Body)) + } +} + +type DocumentBlob struct { + blob []byte +} + +func (d *DocumentBlob) UnmarshalJSON(data []byte) error { + d.blob = make([]byte, len(data)) + copy(d.blob, data) + return nil +} + +func (d *DocumentBlob) MarshalJSON() ([]byte, error) { + return d.blob, nil +} + +type VespaVisitOutput struct { + PathId string `json:"pathId"` + Documents []DocumentBlob `json:"documents"` + DocumentCount int `json:"documentCount"` + Continuation string `json:"continuation"` + ErrorMsg string `json:"message"` +} + +func parseVisitOutput(r io.Reader) (*VespaVisitOutput, error) { + codec := json.NewDecoder(r) + var parsedJson VespaVisitOutput + err := codec.Decode(&parsedJson) + if err != nil { + return nil, fmt.Errorf("could not decode JSON, error: %s", err.Error()) + } + return &parsedJson, nil +} diff --git a/client/go/internal/cli/cmd/visit_test.go b/client/go/internal/cli/cmd/visit_test.go new file mode 100644 index 00000000000..4302680b9d9 --- /dev/null +++ b/client/go/internal/cli/cmd/visit_test.go @@ -0,0 +1,142 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package cmd + +import ( + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/mock" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +const ( + normalpre = `{"pathId":"/document/v1/","documents":[` + document1 = `{"id":"id:t:m::1","fields":{"title":"t"}}` + document2 = `{"id":"id:t:m::2","fields":{"title":"t2"}}` + document3 = `{"id":"id:t:m::3","fields":{"ar":"xyz","w":63,"title":"xyzzy","year":2000}}` + + savedresponse = `{"pathId":"/document/v1/","documents":[{"id":"id:test:music::1921492307","fields":{"title":"song","year":2010}},{"id":"id:test:music::p_try-this-clean-bonus-dvd-_music_1922003403","fields":{"artist":"xyz","weight":600000,"song":"hate","title":"xyz","year":2000}}],"documentCount":2,"continuation":"AAAACAAAAAAAAAAJAAAAAAAAAAgAAAAAAAABAAAAAAEgAAAAAAAAEAAAAAAAAAAA"}` + + saveddoc0 = `{"id":"id:test:music::1921492307","fields":{"title":"song","year":2010}}` + saveddoc1 = `{"id":"id:test:music::p_try-this-clean-bonus-dvd-_music_1922003403","fields":{"artist":"xyz","weight":600000,"song":"hate","title":"xyz","year":2000}}` + handlersResponse = `{ + "handlers" : [ { + "id" : "com.yahoo.container.usability.BindingsOverviewHandler", + "class" : "com.yahoo.container.usability.BindingsOverviewHandler", + "bundle" : "container-disc:8.0.0", + "serverBindings" : [ "http://*/" ] + }, { + "id" : "com.yahoo.document.restapi.resource.DocumentV1ApiHandler", + "class" : "com.yahoo.document.restapi.resource.DocumentV1ApiHandler", + "bundle" : "vespaclient-container-plugin:8.0.0", + "serverBindings" : [ "http://*/document/v1/*", "http://*/document/v1/*/" ] + } ] +}` + clusterStarResponse = `{"pathId":"/document/v1/","message":"Your Vespa deployment has no content cluster '*', only 'fooCC'"}` +) + +func TestQuoteFunc(t *testing.T) { + var buf []byte = make([]byte, 3) + buf[0] = 'a' + buf[2] = 'z' + for i := 0; i < 256; i++ { + buf[1] = byte(i) + s := string(buf) + res := quoteArgForUrl(s) + if i < 32 || i > 127 { + assert.Equal(t, "a+z", res) + } else { + fmt.Printf("res %3d => '%s'\n", i, res) + } + } +} + +// low-level (unit) test +func TestRunOneVisit(t *testing.T) { + withResponse := func(client *mock.HTTPClient) { + client.NextResponseString(200, savedresponse) + } + op := func(service *vespa.Service) { + vArgs := visitArgs{ + contentCluster: "fooCC", + } + vvo, res := runOneVisit(&vArgs, service, "BBBB") + assert.Equal(t, true, res.Success) + assert.Equal(t, "visited fooCC", res.Message) + assert.Equal(t, "/document/v1/", vvo.PathId) + assert.Equal(t, "", vvo.ErrorMsg) + assert.Equal(t, "AAAACAAAAAAAAAAJAAAAAAAAAAgAAAAAAAABAAAAAAEgAAAAAAAAEAAAAAAAAAAA", vvo.Continuation) + assert.Equal(t, 2, vvo.DocumentCount) + assert.Equal(t, 2, len(vvo.Documents)) + assert.Equal(t, saveddoc0, string(vvo.Documents[0].blob)) + assert.Equal(t, saveddoc1, string(vvo.Documents[1].blob)) + } + req := withMockClient(t, withResponse, op) + assert.Equal(t, "cluster=fooCC&continuation=BBBB", req.URL.RawQuery) + + op = func(service *vespa.Service) { + vArgs := visitArgs{ + contentCluster: "search", + fieldSet: "[id]", + selection: "music.year>2000", + chunkCount: 123, + } + vvo, res := runOneVisit(&vArgs, service, "asdf") + assert.Equal(t, true, res.Success) + assert.Equal(t, 2, vvo.DocumentCount) + } + req = withMockClient(t, withResponse, op) + assert.Equal(t, "cluster=search&fieldSet=%5Bid%5D&selection=music%2Eyear%3E2000&continuation=asdf&wantedDocumentCount=123", req.URL.RawQuery) +} + +func withMockClient(t *testing.T, prepCli func(*mock.HTTPClient), runOp func(*vespa.Service)) *http.Request { + client := &mock.HTTPClient{} + prepCli(client) + cli, _, _ := newTestCLI(t) + cli.httpClient = client + service, _ := documentService(cli) + runOp(service) + return client.LastRequest +} + +func TestVisitCommand(t *testing.T) { + assertVisitResults( + []string{ + "visit", + "--json-lines", + }, + t, + []string{ + normalpre + + document1 + + `],"documentCount":1,"continuation":"CAFE"}`, + normalpre + + document2 + + "," + + document3 + + `],"documentCount":2}`, + }, + "cluster=fooCC&continuation=CAFE&wantedDocumentCount=1000", + document1+"\n"+ + document2+"\n"+ + document3+"\n") +} + +func assertVisitResults(arguments []string, t *testing.T, responses []string, queryPart, output string) { + client := &mock.HTTPClient{} + client.NextResponseString(200, handlersResponse) + client.NextResponseString(400, clusterStarResponse) + for _, resp := range responses { + client.NextResponseString(200, resp) + } + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = client + assert.Nil(t, cli.Run(arguments...)) + assert.Equal(t, output, stdout.String()) + assert.Equal(t, "", stderr.String()) + assert.Equal(t, queryPart, client.LastRequest.URL.RawQuery) + assert.Equal(t, "/document/v1/", client.LastRequest.URL.Path) + assert.Equal(t, "GET", client.LastRequest.Method) +} diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 82842a8c28b..14b39867348 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -38,8 +38,8 @@ <aopalliance.version>1.0</aopalliance.version> <guava.version>27.1-jre</guava.version> <guice.version>4.2.3</guice.version> - <jackson2.version>2.13.4</jackson2.version> - <jackson-databind.version>2.13.4.2</jackson-databind.version> + <jackson2.version>2.14.2</jackson2.version> + <jackson-databind.version>2.14.2</jackson-databind.version> <javax.inject.version>1</javax.inject.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <javax.ws.rs-api.version>2.0.1</javax.ws.rs-api.version> diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 5707206019f..f5ad1dce4e7 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -106,6 +106,8 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { private final boolean includeSourceFiles; private final TransformerFactory transformerFactory; + private DeploymentSpec deploymentSpec = null; + /** Creates from a directory with source files included */ public static FilesApplicationPackage fromFile(File appDir) { return fromFile(appDir, false); @@ -580,6 +582,12 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { IOUtils.writeFile(metaFile, metaData.asJsonBytes()); } + @Override + public DeploymentSpec getDeploymentSpec() { + if (deploymentSpec != null) return deploymentSpec; + return deploymentSpec = parseDeploymentSpec(false); + } + private void preprocessXML(File destination, File inputXml, Zone zone) throws IOException { if ( ! inputXml.exists()) return; try { @@ -589,10 +597,9 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { instance, zone.environment(), zone.region(), - getDeployment().map(new DeploymentSpecXmlReader(false)::read) - .flatMap(spec -> spec.instance(instance)) - .map(DeploymentInstanceSpec::tags) - .orElse(Tags.empty())) + getDeploymentSpec().instance(instance) + .map(DeploymentInstanceSpec::tags) + .orElse(Tags.empty())) .run(); try (FileOutputStream outputStream = new FileOutputStream(destination)) { diff --git a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java index 93e038c786a..19f1414cd10 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java @@ -110,10 +110,7 @@ public class HostedOverrideProcessorComplexTest { private void assertOverride(InstanceName instance, Environment environment, RegionName region, String expected) throws TransformerException { ApplicationPackage app = FilesApplicationPackage.fromFile(new File(servicesFile).getParentFile()); Document inputDoc = Xml.getDocument(app.getServices()); - Tags tags = app.getDeployment() - .map(new DeploymentSpecXmlReader(false)::read) - .flatMap(spec -> spec.instance(instance).map(DeploymentInstanceSpec::tags)) - .orElse(Tags.empty()); + Tags tags = app.getDeploymentSpec().instance(instance).map(DeploymentInstanceSpec::tags).orElse(Tags.empty()); Document newDoc = new OverrideProcessor(instance, environment, region, tags).process(inputDoc); assertEquals(expected, Xml.documentAsString(newDoc, true)); } diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java index 6c83b2029ad..4742d275918 100644 --- a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java @@ -84,6 +84,7 @@ public class FilesApplicationPackageTest { assertFalse(new File(appDir, "deployment.xml").exists()); FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir); assertFalse(app.getDeployment().isPresent()); + assertTrue(app.getDeploymentSpec().isEmpty()); } @Test @@ -93,6 +94,7 @@ public class FilesApplicationPackageTest { assertTrue(deployment.exists()); FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir); assertTrue(app.getDeployment().isPresent()); + assertFalse(app.getDeploymentSpec().isEmpty()); assertFalse(app.getMajorVersion().isPresent()); assertEquals(IOUtils.readAll(app.getDeployment().get()), IOUtils.readAll(new FileReader(deployment))); } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index 0ce09c454a0..b6a183c06ab 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -2,6 +2,7 @@ package com.yahoo.config.application.api; import com.yahoo.component.Version; +import com.yahoo.config.application.api.xml.DeploymentSpecXmlReader; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; @@ -128,20 +129,7 @@ public interface ApplicationPackage { /** Returns the major version this application is valid for, or empty if it is valid for all versions */ default Optional<Integer> getMajorVersion() { - if (getDeployment().isEmpty()) return Optional.empty(); - - Element deployElement = XML.getDocument(getDeployment().get()).getDocumentElement(); - if (deployElement == null) return Optional.empty(); - - String majorVersionString = deployElement.getAttribute("major-version"); - if (majorVersionString == null || majorVersionString.isEmpty()) - return Optional.empty(); - try { - return Optional.of(Integer.parseInt(majorVersionString)); - } - catch (NumberFormatException e) { - throw new IllegalArgumentException("major-version must be an integer number, not '" + majorVersionString + "'"); - } + return getDeploymentSpec().majorVersion(); } /** @@ -168,6 +156,19 @@ public interface ApplicationPackage { String getServicesSource(); Optional<Reader> getDeployment(); + + /** + * Returns the parsed deployment spec of this, + * without validating it, and without reparsing on each request. + */ + DeploymentSpec getDeploymentSpec(); + + default DeploymentSpec parseDeploymentSpec(boolean validate) { + return getDeployment() + .map(new DeploymentSpecXmlReader(validate)::read) + .orElse(DeploymentSpec.empty); + } + Optional<Reader> getValidationOverrides(); List<ComponentInfo> getComponentsInfo(Version vespaVersion); @@ -226,6 +227,7 @@ public interface ApplicationPackage { /** * Readers for all the schema files. + * * @return a collection of readers for schemas */ Collection<NamedReader> getSchemas(); @@ -235,10 +237,9 @@ public interface ApplicationPackage { * application package. This is the entry point for the multi environment application package support. This method * will not mutate the existing application package. * - * @param zone A valid {@link Zone} instance, used to decide which parts of services to keep and remove - * @param logger A {@link DeployLogger} to add output that will be returned to the user - * - * @return A new application package instance pointing to a new location + * @param zone a valid {@link Zone} instance, used to decide which parts of services to keep and remove + * @param logger a {@link DeployLogger} to add output that will be returned to the user + * @return a new application package instance pointing to a new location */ default ApplicationPackage preprocess(Zone zone, DeployLogger logger) throws IOException { throw new UnsupportedOperationException("This application package does not support preprocessing"); diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/Bcp.java b/config-model-api/src/main/java/com/yahoo/config/application/api/Bcp.java index 8f88b1ba74c..48464904f44 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/Bcp.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/Bcp.java @@ -87,27 +87,18 @@ public class Bcp { public static class Group { private final List<RegionMember> members; - private final List<Endpoint> endpoints; + private final Set<RegionName> memberRegions; private final Duration deadline; public Group(List<RegionMember> members, Duration deadline) { - this(members, List.of(), deadline); - } - - public Group(List<RegionMember> members, List<Endpoint> endpoints, Duration deadline) { this.members = List.copyOf(members); - this.endpoints = endpoints; + this.memberRegions = members.stream().map(member -> member.region()).collect(Collectors.toSet()); this.deadline = deadline; } public List<RegionMember> members() { return members; } - /** - * Returns the endpoints defined in this. - * These will be added to instances during XML import post processing - * and should not otherwise be exposed from here. - */ - List<Endpoint> endpoints() { return endpoints; } + public Set<RegionName> memberRegions() { return memberRegions; } /** * Returns the max time until the other regions must be able to handle the additional traffic diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index 41644ebc87d..699010417bf 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -88,6 +88,8 @@ public class DeploymentSpec { validateBcp(); } + public boolean isEmpty() { return this == empty; } + /** Throw an IllegalArgumentException if the total delay exceeds 24 hours */ private void validateTotalDelay(List<Step> steps) { long totalDelaySeconds = steps.stream().mapToLong(step -> (step.delay().getSeconds())).sum(); diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index 91cb2f2622e..d04bb7ecfe0 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -519,7 +519,7 @@ public class DeploymentSpecXmlReader { } Duration deadline = XML.attribute("deadline", groupElement).map(value -> toDuration(value, "deadline")).orElse(Duration.ZERO); - groups.add(new Bcp.Group(regions, endpoints, deadline)); + groups.add(new Bcp.Group(regions, deadline)); } validateAndConsolidate(endpointsByZone, zoneEndpoints); return new Bcp(groups); diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 04de4d2e277..d5c074a191d 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -111,6 +111,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"tokle"}) default boolean useRestrictedDataPlaneBindings() { return false; } @ModelFeatureFlag(owners = {"arnej","baldersheim"}, removeAfter = "8.110") default boolean useOldJdiscContainerStartup() { return false; } @ModelFeatureFlag(owners = {"tokle, bjorncs"}, removeAfter = "8.108") default boolean enableDataPlaneFilter() { return true; } + @ModelFeatureFlag(owners = {"arnej, bjorncs"}) default boolean enableGlobalPhase() { return false; } //Below are all flags that must be kept until 7 is out of the door @ModelFeatureFlag(owners = {"arnej"}, removeAfter="7.last") default boolean ignoreThreadStackSizes() { return false; } diff --git a/config-model/.gitignore b/config-model/.gitignore index 6edd041cbe8..7e7e597675d 100644 --- a/config-model/.gitignore +++ b/config-model/.gitignore @@ -4,5 +4,6 @@ /target /src/test/integration/*/copy/ /src/test/integration/*/models.generated/ +/src/test/derived/*/models.generated/ *.cfg.actual /var/ diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index 7c805417739..f2d63f3bba4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -102,21 +102,6 @@ public class ApplicationConfigProducerRoot extends TreeConfigProducer<AnyConfigP } /** - * Returns the Service with the given id, or null if no such - * configId exists or if it belongs to a non-Service ConfigProducer. - * - * @param configId The configId, e.g. "search.0/tld.0" - * @return Service with the given configId - */ - public Service getService(String configId) { - ConfigProducer cp = getConfigProducer(configId); - if (cp == null || !(cp instanceof Service)) { - return null; - } - return (Service) cp; - } - - /** * Adds the descendant (at any depth level), so it can be looked up * on configId in the Map. * diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java index 2685570b444..051591baa75 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java @@ -51,7 +51,7 @@ public abstract class ConfigModel { * * @param configModelRepo The ConfigModelRepo of the system model */ - public void prepare(ConfigModelRepo configModelRepo, DeployState deployState) { return; } + public void prepare(ConfigModelRepo configModelRepo, DeployState deployState) { } /** * <p>Returns whether this model must be maintained in memory for serving config requests. diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java index d9918168266..13d87b852e4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java @@ -7,8 +7,11 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.vespa.model.VespaModel; +import java.time.Duration; +import java.util.Comparator; import java.util.stream.Stream; /** @@ -67,6 +70,18 @@ public final class ConfigModelContext { return ConfigModelContext.create(deployState, vespaModel, configModelRepoAdder, parent, producerId); } + /** Returns a cluster info builder pre-populated with info known in this context. */ + public ClusterInfo.Builder clusterInfo() { + var instance = getApplicationPackage().getDeploymentSpec().instance(properties().applicationId().instance()); + if ( ! instance.isPresent()) return new ClusterInfo.Builder(); + var maxDeadline = instance.get().bcp().groups().stream() + .filter(group -> group.memberRegions().contains(properties().zone().region())) + .map(group -> group.deadline()) + .min(Comparator.comparing(deadline -> deadline)) + .orElse(Duration.ofMinutes(0)); + return new ClusterInfo.Builder().bcpDeadline(maxDeadline); + } + /** * Create an application context from a parent producer and an id. * diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java index 45f64182b2a..8c72b5c0237 100644 --- a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java @@ -88,10 +88,9 @@ public abstract class ConfigModelBuilder<MODEL extends ConfigModel> extends Abst @Override public boolean equals(Object other) { - if (!(other instanceof ConfigModelBuilder)) { + if (!(other instanceof ConfigModelBuilder<?> otherBuilder)) { return false; } - ConfigModelBuilder<?> otherBuilder = (ConfigModelBuilder<?>) other; List<ConfigModelId> thisIds = this.handlesElements(); List<ConfigModelId> otherIds = otherBuilder.handlesElements(); if (thisIds.size() != otherIds.size()) { diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java index cd283866550..e1970b001e1 100644 --- a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java @@ -42,8 +42,7 @@ public class ConfigModelId implements Comparable<ConfigModelId> { @Override public boolean equals(Object object) { - if (!(object instanceof ConfigModelId)) return false; - ConfigModelId other = (ConfigModelId)object; + if (!(object instanceof ConfigModelId other)) return false; return this.name.equals(other.name) && this.version.equals(other.version); } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 1813e183a60..b8d63ba3778 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -24,6 +24,7 @@ import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.provision.HostsXmlProvisioner; +import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.provision.SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.DockerImage; @@ -77,7 +78,7 @@ public class DeployState implements ConfigDefinitionStore { private final ModelContext.Properties properties; private final Version vespaVersion; private final Set<ContainerEndpoint> endpoints; - private final Zone zone; + private final Zone zone; // TODO: Zone is set separately both here and in properties private final QueryProfiles queryProfiles; private final SemanticRules semanticRules; private final ImportedMlModels importedModels; diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 49194a5d1bb..ecbb990f096 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -137,6 +137,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean useRestrictedDataPlaneBindings() { return useRestrictedDataPlaneBindings; } @Override public Optional<CloudAccount> cloudAccount() { return cloudAccount; } @Override public boolean allowUserFilters() { return allowUserFilters; } + @Override public boolean enableGlobalPhase() { return true; } // Enable global-phase by default for unit tests only public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim; diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AnyConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AnyConfigProducer.java index cd21fccd855..547e81354eb 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/AnyConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AnyConfigProducer.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.producer; -import com.yahoo.api.annotations.Beta; import com.yahoo.config.ConfigInstance; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.deploy.DeployState; @@ -16,11 +15,8 @@ import com.yahoo.vespa.model.HostSystem; import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.monitoring.Monitoring; - import java.io.Serializable; -import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/TreeConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/TreeConfigProducer.java index 012bffaf7f6..30f9cd202ff 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/TreeConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/TreeConfigProducer.java @@ -3,19 +3,14 @@ package com.yahoo.config.model.producer; import com.yahoo.api.annotations.Beta; import com.yahoo.config.model.ApplicationConfigProducerRoot; -import com.yahoo.vespa.model.ConfigProducer; import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.SimpleConfigProducer; import com.yahoo.vespa.model.utils.FreezableMap; -import java.io.PrintStream; -import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; /** * Superclass for all producers with children. diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java index b351073bd25..5ea22ee4d25 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java @@ -36,9 +36,6 @@ public class Hosts { for (Host host : hosts) hostsBuilder.put(host.hostname(), host); this.hosts = hostsBuilder.build(); - - // Don't limit zk connections on non-hosted systems - System.setProperty("zookeeper.vespa.clients", ""); } /** Throw IllegalArgumentException if host aliases breaks invariants */ diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index dd6087eefc7..41697e61bf2 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -129,6 +129,8 @@ public class InMemoryProvisioner implements HostProvisioner { this.retiredHostNames = Set.of(retiredHostNames); } + public Provisioned provisioned() { return provisioned; } + /** May affect e.g. the number of nodes/cluster. */ public InMemoryProvisioner setEnvironment(Environment environment) { this.environment = environment; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 9ee279c68d3..3b715c63105 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -6,6 +6,7 @@ import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; @@ -57,12 +58,14 @@ public class MockApplicationPackage implements ApplicationPackage { private final List<String> schemas; private final Map<Path, MockApplicationFile> files; private final String schemaDir; - private final Optional<String> deploymentSpec; + private final Optional<String> deploymentSpecString; private final Optional<String> validationOverrides; private final boolean failOnValidateXml; private final QueryProfileRegistry queryProfileRegistry; private final ApplicationMetaData applicationMetaData; + private DeploymentSpec deploymentSpec = null; + protected MockApplicationPackage(File root, String hosts, String services, List<String> schemas, Map<Path, MockApplicationFile> files, String schemaDir, @@ -74,7 +77,7 @@ public class MockApplicationPackage implements ApplicationPackage { this.schemas = schemas; this.files = files; this.schemaDir = schemaDir; - this.deploymentSpec = Optional.ofNullable(deploymentSpec); + this.deploymentSpecString = Optional.ofNullable(deploymentSpec); this.validationOverrides = Optional.ofNullable(validationOverrides); this.failOnValidateXml = failOnValidateXml; queryProfileRegistry = new QueryProfileXMLReader().read(asNamedReaderList(queryProfileType), @@ -102,6 +105,12 @@ public class MockApplicationPackage implements ApplicationPackage { } @Override + public DeploymentSpec getDeploymentSpec() { + if (deploymentSpec != null) return deploymentSpec; + return deploymentSpec = parseDeploymentSpec(false); + } + + @Override public Reader getHosts() { if (hostsS == null) return null; return new StringReader(hostsS); @@ -183,7 +192,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public Optional<Reader> getDeployment() { - return deploymentSpec.map(StringReader::new); + return deploymentSpecString.map(StringReader::new); } @Override @@ -215,13 +224,6 @@ public class MockApplicationPackage implements ApplicationPackage { return new MockApplicationPackage.Builder().withHosts(emptyHosts).withServices(emptyServices).build(); } - public static ApplicationPackage fromSearchDefinitionDirectory(String dir) { - return new MockApplicationPackage.Builder() - .withEmptyHosts() - .withEmptyServices() - .withSchemaDir(dir).build(); - } - // TODO: It might work to just merge this and the above public static ApplicationPackage fromSearchDefinitionAndRootDirectory(String dir) { return new MockApplicationPackage.Builder() diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java index 7b2aaa32136..365434b9de5 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java @@ -7,7 +7,6 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelRepo; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AnyConfigProducer; -import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.vespa.model.ConfigProducer; import com.yahoo.vespa.model.HostSystem; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/ModelBuilderAddingAccessControlFilter.java b/config-model/src/main/java/com/yahoo/config/model/test/ModelBuilderAddingAccessControlFilter.java index 840d781ac9c..234aecc6228 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/ModelBuilderAddingAccessControlFilter.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/ModelBuilderAddingAccessControlFilter.java @@ -45,8 +45,7 @@ public class ModelBuilderAddingAccessControlFilter } private static void addFilterToContainerCluster(ContainerModel containerModel) { - if (!(containerModel.getCluster() instanceof ApplicationContainerCluster)) return; - ApplicationContainerCluster cluster = (ApplicationContainerCluster) containerModel.getCluster(); + if (!(containerModel.getCluster() instanceof ApplicationContainerCluster cluster)) return; Http http = cluster.getHttp(); if (http.getAccessControl().isPresent()) { Chain<Filter> chain = http.getFilterChains() diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java b/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java index fd98d21dcd5..7aca60bb930 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java @@ -21,7 +21,6 @@ import java.util.List; * xml string and returns a config producer that can be use to test getConfig. * * @author Ulf Lilleengen - * @since 5.1.20 */ @Beta public class TestDriver { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java index c1fd8e4646d..f243c4635c7 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java @@ -13,7 +13,6 @@ import java.util.List; * Test utility class that provides many methods for inspecting the state of a completely built model * * @author Ulf Lilleengen - * @since 5.1 */ @Beta public class TestRoot { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java b/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java index c05d7bf4942..24c418a9d2f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java @@ -4,8 +4,6 @@ package com.yahoo.config.model.test; import com.yahoo.collections.CollectionUtil; import com.yahoo.config.model.builder.xml.XmlHelper; import org.w3c.dom.Element; -import org.xml.sax.InputSource; - import java.io.StringReader; import java.util.ArrayList; import java.util.Arrays; @@ -35,7 +33,4 @@ public class TestUtil { return String.join("\n", lines); } - private static InputSource inputSource(String str) { - return new InputSource(new StringReader(str)); - } } diff --git a/config-model/src/main/java/com/yahoo/schema/OnnxModel.java b/config-model/src/main/java/com/yahoo/schema/OnnxModel.java index ae6f1fd96e4..272b668b5fb 100644 --- a/config-model/src/main/java/com/yahoo/schema/OnnxModel.java +++ b/config-model/src/main/java/com/yahoo/schema/OnnxModel.java @@ -3,6 +3,7 @@ package com.yahoo.schema; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.model.ml.OnnxModelInfo; +import com.yahoo.searchlib.rankingexpression.Reference; import java.util.Collections; import java.util.HashMap; @@ -44,11 +45,37 @@ public class OnnxModel extends DistributableResource { addInputNameMapping(onnxName, vespaName, true); } + private String validateInputSource(String source) { + var optRef = Reference.simple(source); + if (optRef.isPresent()) { + Reference ref = optRef.get(); + // input can be one of: + // attribute(foo), query(foo), constant(foo) + if (FeatureNames.isSimpleFeature(ref)) { + return ref.toString(); + } + // or a function (evaluated by backend) + if (ref.isSimpleRankingExpressionWrapper()) { + var arg = ref.simpleArgument(); + if (arg.isPresent()) { + return ref.toString(); + } + } + } else { + // otherwise it must be an identifier + Reference ref = Reference.fromIdentifier(source); + return ref.toString(); + } + // invalid input source + throw new IllegalArgumentException("invalid input for ONNX model " + getName() + ": " + source); + } + public void addInputNameMapping(String onnxName, String vespaName, boolean overwrite) { Objects.requireNonNull(onnxName, "Onnx name cannot be null"); Objects.requireNonNull(vespaName, "Vespa name cannot be null"); + String source = validateInputSource(vespaName); if (overwrite || ! inputMap.containsKey(onnxName)) { - inputMap.put(onnxName, vespaName); + inputMap.put(onnxName, source); } } @@ -59,8 +86,10 @@ public class OnnxModel extends DistributableResource { public void addOutputNameMapping(String onnxName, String vespaName, boolean overwrite) { Objects.requireNonNull(onnxName, "Onnx name cannot be null"); Objects.requireNonNull(vespaName, "Vespa name cannot be null"); + // output name must be a valid identifier: + var ref = Reference.fromIdentifier(vespaName); if (overwrite || ! outputMap.containsKey(onnxName)) { - outputMap.put(onnxName, vespaName); + outputMap.put(onnxName, ref.toString()); } } diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java index ad6eb038058..a00bbb682a8 100644 --- a/config-model/src/main/java/com/yahoo/schema/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java @@ -1019,6 +1019,9 @@ public class RankProfile implements Cloneable { var recorder = new InputRecorder(needInputs); recorder.transform(globalPhaseRanking.function().getBody(), context); for (String input : needInputs) { + if (input.startsWith("constant(") || input.startsWith("query(")) { + continue; + } try { addMatchFeatures(new FeatureList(input)); } catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) { @@ -1166,7 +1169,7 @@ public class RankProfile implements Cloneable { // Source is either a simple reference (query/attribute/constant/rankingExpression)... Optional<Reference> reference = Reference.simple(source); if (reference.isPresent()) { - if (reference.get().name().equals("rankingExpression") && reference.get().simpleArgument().isPresent()) { + if (reference.get().isSimpleRankingExpressionWrapper()) { source = reference.get().simpleArgument().get(); // look up function below } else { return Optional.of(context.getType(reference.get())); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java index 31a38752bec..6272563f833 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java @@ -20,6 +20,7 @@ import com.yahoo.searchlib.rankingexpression.parser.ParseException; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; import com.yahoo.vespa.config.search.RankProfilesConfig; +import static com.yahoo.searchlib.rankingexpression.Reference.wrapInRankingExpression; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -247,15 +248,15 @@ public class RawRankProfile implements RankProfilesConfig.Producer { SerializationContext context) { for (Map.Entry<String, RankProfile.RankingExpressionFunction> e : functions.entrySet()) { String propertyName = RankingExpression.propertyName(e.getKey()); - if (context.serializedFunctions().containsKey(propertyName)) continue; + if (! context.serializedFunctions().containsKey(propertyName)) { - String expressionString = e.getValue().function().getBody().getRoot().toString(context).toString(); + String expressionString = e.getValue().function().getBody().getRoot().toString(context).toString(); + context.addFunctionSerialization(propertyName, expressionString); + e.getValue().function().argumentTypes().entrySet().stream().sorted(Map.Entry.comparingByKey()) + .forEach(argumentType -> context.addArgumentTypeSerialization(e.getKey(), argumentType.getKey(), argumentType.getValue())); + } + e.getValue().function().returnType().ifPresent(t -> context.addFunctionTypeSerialization(e.getKey(), t)); - context.addFunctionSerialization(propertyName, expressionString); - e.getValue().function().argumentTypes().entrySet().stream().sorted(Map.Entry.comparingByKey()) - .forEach(argumentType -> context.addArgumentTypeSerialization(e.getKey(), argumentType.getKey(), argumentType.getValue())); - if (e.getValue().function().returnType().isPresent()) - context.addFunctionTypeSerialization(e.getKey(), e.getValue().function().returnType().get()); // else if (e.getValue().function().arguments().isEmpty()) TODO: Enable this check when we resolve all types // throw new IllegalStateException("Type of function '" + e.getKey() + "' is not resolved"); } @@ -273,9 +274,10 @@ public class RawRankProfile implements RankProfilesConfig.Producer { String propertyName = RankingExpression.propertyName(referenceNode.getName()); String expressionString = function.getBody().getRoot().toString(context).toString(); context.addFunctionSerialization(propertyName, expressionString); - ReferenceNode backendReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", - referenceNode.getArguments().expressions(), - referenceNode.getOutput()); + function.returnType().ifPresent(t -> context.addFunctionTypeSerialization(referenceNode.getName(), t)); + var backendReferenceNode = new ReferenceNode(wrapInRankingExpression(referenceNode.getName()), + referenceNode.getArguments().expressions(), + referenceNode.getOutput()); // tell backend to map back to the name the user expects: featureRenames.put(backendReferenceNode.toString(), referenceNode.toString()); functionFeatures.put(referenceNode.getName(), backendReferenceNode); @@ -499,7 +501,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { if (expression.getRoot() instanceof ReferenceNode) { properties.add(new Pair<>("vespa.rank." + phase, expression.getRoot().toString())); } else { - properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + name + ")")); + properties.add(new Pair<>("vespa.rank." + phase, wrapInRankingExpression(name))); properties.add(new Pair<>(RankingExpression.propertyName(name), expression.getRoot().toString())); } return properties; @@ -520,7 +522,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { for (Map.Entry<String, String> mapping : onnxModel.getInputMap().entrySet()) { String source = mapping.getValue(); if (functionNames.contains(source)) { - onnxModel.addInputNameMapping(mapping.getKey(), "rankingExpression(" + source + ")"); + onnxModel.addInputNameMapping(mapping.getKey(), wrapInRankingExpression(source)); } } } diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java index b0f63ebb732..4e7988a2006 100644 --- a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java @@ -3,16 +3,13 @@ package com.yahoo.schema.expressiontransforms; import com.yahoo.schema.FeatureNames; import com.yahoo.schema.RankProfile; -import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.searchlib.rankingexpression.parser.ParseException; import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; -import java.io.StringReader; import java.util.Set; /** @@ -86,13 +83,7 @@ public class InputRecorder extends ExpressionTransformer<RankProfileTransformCon throw new IllegalArgumentException("missing onnx model: " + arg); } for (String onnxInput : model.getInputMap().values()) { - var reader = new StringReader(onnxInput); - try { - var asExpression = new RankingExpression(reader); - transform(asExpression.getRoot(), context); - } catch (ParseException e) { - throw new IllegalArgumentException("illegal onnx input '" + onnxInput + "': " + e.getMessage()); - } + neededInputs.add(onnxInput); } return; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Host.java b/config-model/src/main/java/com/yahoo/vespa/model/Host.java index 7dbab87fac0..047a6ef9bd5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/Host.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/Host.java @@ -34,8 +34,6 @@ public final class Host extends TreeConfigProducer<AnyConfigProducer> implements Objects.requireNonNull(hostname, "The host name of a host cannot be null"); this.runsConfigServer = runsConfigServer; this.hostname = hostname; - if (parent instanceof HostSystem) - ((HostSystem)parent).checkName(hostname); } public static Host createConfigServerHost(HostSystem hostSystem, String hostname) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java b/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java index 125966ae91d..2f704db1862 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java @@ -22,11 +22,7 @@ public class HostPorts { public final static int BASE_PORT = 19100; final static int MAX_PORTS = 799; - private DeployLogger deployLogger = new DeployLogger() { - public void log(Level level, String message) { - System.err.println("deploy log["+level+"]: "+message); - } - }; + private DeployLogger deployLogger = (level, message) -> System.err.println("deploy log["+level+"]: "+message); private final Map<Integer, NetworkPortRequestor> portDB = new LinkedHashMap<>(); @@ -98,7 +94,7 @@ public class HostPorts { /** Allocate a specific port number for a service */ public int requireNetworkPort(int port, NetworkPortRequestor service, String suffix) { - reservePort(service, port, suffix); + reservePort(service, port); String servType = service.getServiceType(); String configId = service.getConfigId(); portFinder.use(new NetworkPorts.Allocation(port, servType, configId, suffix)); @@ -119,18 +115,13 @@ public class HostPorts { return requireNetworkPort(port, service, suffix); } - /** Convenience method to allocate a preferred or required port number for a service */ - public int wantNetworkPort(int port, NetworkPortRequestor service, String suffix, boolean forceRequired) { - return forceRequired ? requireNetworkPort(port, service, suffix) : wantNetworkPort(port, service, suffix); - } - /** Allocate a dynamic port number for a service */ public int allocateNetworkPort(NetworkPortRequestor service, String suffix) { String servType = service.getServiceType(); String configId = service.getConfigId(); int fallback = nextAvailableNetworkPort(); int port = portFinder.findPort(new NetworkPorts.Allocation(fallback, servType, configId, suffix), hostname); - reservePort(service, port, suffix); + reservePort(service, port); portFinder.use(new NetworkPorts.Allocation(port, servType, configId, suffix)); return port; } @@ -161,7 +152,7 @@ public class HostPorts { * @param service the service that wishes to reserve the port. * @param port the port to be reserved. */ - void reservePort(NetworkPortRequestor service, int port, String suffix) { + void reservePort(NetworkPortRequestor service, int port) { if (portDB.containsKey(port)) { portAlreadyReserved(service, port); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index 53e2ce0e652..00a1078b294 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -10,6 +10,7 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.ProvisionLogger; + import java.net.UnknownHostException; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -53,10 +54,11 @@ public class HostSystem extends TreeConfigProducer<Host> { this.isHosted = isHosted; } - void checkName(String hostname) { + String checkHostname(String hostname) { + if (isHosted) return hostname; // Done in node-repo instead + if (doCheckIp) { - // Bad DNS config in a hosted system isn't actionable by the tenant, so we log any warnings internally - BiConsumer<Level, String> logFunction = isHosted ? deployLogger::log : deployLogger::logApplicationPackage; + BiConsumer<Level, String> logFunction = deployLogger::logApplicationPackage; // Give a warning if the host does not exist try { var inetAddr = java.net.InetAddress.getByName(hostname); @@ -69,6 +71,7 @@ public class HostSystem extends TreeConfigProducer<Host> { logFunction.accept(Level.WARNING, "Unable to lookup IP address of host: " + hostname); } } + return hostname; } @Override @@ -86,10 +89,10 @@ public class HostSystem extends TreeConfigProducer<Host> { } private HostResource addNewHost(HostSpec hostSpec) { - Host host = Host.createHost(this, hostSpec.hostname()); - HostResource hostResource = new HostResource(host, hostSpec); + String hostname = checkHostname(hostSpec.hostname()); + HostResource hostResource = new HostResource(Host.createHost(this, hostname), hostSpec); hostSpec.networkPorts().ifPresent(np -> hostResource.ports().addNetworkPorts(np)); - hostname2host.put(host.getHostname(), hostResource); + hostname2host.put(hostname, hostResource); return hostResource; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 4e40bd768bf..dd1579eae17 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -24,7 +24,6 @@ import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AnyConfigProducer; -import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.config.model.producer.UserConfigRepo; import com.yahoo.config.provision.AllocatedHosts; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index b9d0cf0d17b..5a1c3d87e5e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -25,12 +25,10 @@ import ai.vespa.metricsproxy.telegraf.TelegrafConfig; import ai.vespa.metricsproxy.telegraf.TelegrafRegistry; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.TreeConfigProducer; -import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.osgi.provider.model.ComponentModel; -import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.monitoring.MetricSet; import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java index a3fdce98c73..29a54548256 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.admin.monitoring; import com.yahoo.metrics.ContainerMetrics; import com.yahoo.metrics.SearchNodeMetrics; import com.yahoo.metrics.StorageMetrics; +import com.yahoo.metrics.HostedNodeAdminMetrics; import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; @@ -21,29 +22,30 @@ public class AutoscalingMetrics { private static MetricSet create() { List<String> metrics = new ArrayList<>(); - metrics.add("cpu.util"); + metrics.add(HostedNodeAdminMetrics.CPU_UTIL.baseName()); // Memory util - metrics.add("mem.util"); // node level - default - metrics.add(SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_MEMORY.average()); // better for content as it is the basis for blocking + metrics.add(HostedNodeAdminMetrics.MEM_UTIL.baseName()); // node level - default + metrics.add(SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_MEMORY.average()); // the basis for blocking // Disk util - metrics.add("disk.util"); // node level -default - metrics.add(SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_DISK.average()); // better for content as it is the basis for blocking + metrics.add(HostedNodeAdminMetrics.DISK_UTIL.baseName()); // node level -default + metrics.add(SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_DISK.average()); // the basis for blocking - metrics.add("application_generation"); + metrics.add(ContainerMetrics.APPLICATION_GENERATION.last()); + metrics.add(SearchNodeMetrics.CONTENT_PROTON_CONFIG_GENERATION.last()); - metrics.add("in_service"); + metrics.add(ContainerMetrics.IN_SERVICE.last()); // Query rate - metrics.add(ContainerMetrics.QUERIES.rate()); // container - metrics.add(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_QUERIES.rate()); // content + metrics.add(ContainerMetrics.QUERIES.rate()); + metrics.add(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_QUERIES.rate()); // Write rate - metrics.add(ContainerMetrics.FEED_HTTP_REQUESTS.rate()); // container - metrics.add(StorageMetrics.VDS_FILESTOR_ALLTHREADS_PUT_COUNT.rate()); // content - metrics.add(StorageMetrics.VDS_FILESTOR_ALLTHREADS_REMOVE_COUNT.rate()); // content - metrics.add(StorageMetrics.VDS_FILESTOR_ALLTHREADS_UPDATE_COUNT.rate()); // content + metrics.add(ContainerMetrics.FEED_HTTP_REQUESTS.rate()); + metrics.add(StorageMetrics.VDS_FILESTOR_ALLTHREADS_PUT_COUNT.rate()); + metrics.add(StorageMetrics.VDS_FILESTOR_ALLTHREADS_REMOVE_COUNT.rate()); + metrics.add(StorageMetrics.VDS_FILESTOR_ALLTHREADS_UPDATE_COUNT.rate()); return new MetricSet("autoscaling", toMetrics(metrics)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/NetworkMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/NetworkMetrics.java index 2f9c97f0488..839dcad64ee 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/NetworkMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/NetworkMetrics.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; +import com.yahoo.metrics.HostedNodeAdminMetrics; + import com.google.common.collect.ImmutableSet; import java.util.Set; @@ -8,18 +10,21 @@ import java.util.Set; /** * @author gjoranv */ + +// TODO: Move to hosted repo. public class NetworkMetrics { public static final MetricSet networkMetricSet = createNetworkMetricSet(); private static MetricSet createNetworkMetricSet() { Set<Metric> dockerNetworkMetrics = - ImmutableSet.of(new Metric("net.in.bytes"), - new Metric("net.in.errors"), - new Metric("net.in.dropped"), - new Metric("net.out.bytes"), - new Metric("net.out.errors"), - new Metric("net.out.dropped") + ImmutableSet.of(new Metric(HostedNodeAdminMetrics.NET_IN_BYTES.baseName()), + new Metric(HostedNodeAdminMetrics.NET_IN_ERROR.baseName()), + new Metric(HostedNodeAdminMetrics.NET_IN_DROPPED.baseName()), + new Metric(HostedNodeAdminMetrics.NET_OUT_BYTES.baseName()), + new Metric(HostedNodeAdminMetrics.NET_OUT_ERROR.baseName()), + new Metric(HostedNodeAdminMetrics.NET_OUT_DROPPED.baseName()), + new Metric(HostedNodeAdminMetrics.BANDWIDTH_LIMIT.baseName()) ); return new MetricSet("network", dockerNetworkMetrics); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java index 0958a3f3908..eee6be9af93 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; +import com.yahoo.metrics.HostedNodeAdminMetrics; + import com.google.common.collect.ImmutableSet; import java.util.Set; @@ -8,56 +10,39 @@ import java.util.Set; /** * @author gjoranv */ -public class SystemMetrics { +// TODO: Move to hosted repo. - public static final String CPU_UTIL = "cpu.util"; - public static final String CPU_SYS_UTIL = "cpu.sys.util"; - public static final String CPU_THROTTLED_TIME = "cpu.throttled_time.rate"; - public static final String CPU_THROTTLED_CPU_TIME = "cpu.throttled_cpu_time.rate"; - public static final String CPU_VCPUS = "cpu.vcpus"; - public static final String DISK_LIMIT = "disk.limit"; - public static final String DISK_USED = "disk.used"; - public static final String DISK_UTIL = "disk.util"; - public static final String MEM_LIMIT = "mem.limit"; - public static final String MEM_USED = "mem.used"; - public static final String MEM_UTIL = "mem.util"; - public static final String MEM_TOTAL_USED = "mem_total.used"; - public static final String MEM_TOTAL_UTIL = "mem_total.util"; - public static final String BANDWIDTH_LIMIT = "bandwidth.limit"; - public static final String GPU_UTIL = "gpu.util"; - public static final String GPU_MEM_USED = "gpu.memory.used"; - public static final String GPU_MEM_TOTAL = "gpu.memory.total"; +public class SystemMetrics { public static final MetricSet systemMetricSet = createSystemMetricSet(); private static MetricSet createSystemMetricSet() { Set<Metric> dockerNodeMetrics = - ImmutableSet.of(new Metric(CPU_UTIL), - new Metric(CPU_SYS_UTIL), - new Metric(CPU_THROTTLED_TIME), - new Metric(CPU_THROTTLED_CPU_TIME), - new Metric(CPU_VCPUS), - new Metric(DISK_LIMIT), - new Metric(DISK_USED), - new Metric(DISK_UTIL), - new Metric(MEM_LIMIT), - new Metric(MEM_USED), - new Metric(MEM_UTIL), - new Metric(MEM_TOTAL_USED), - new Metric(MEM_TOTAL_UTIL), - new Metric(BANDWIDTH_LIMIT), - new Metric(GPU_UTIL), - new Metric(GPU_MEM_USED), - new Metric(GPU_MEM_TOTAL) + ImmutableSet.of(new Metric(HostedNodeAdminMetrics.CPU_UTIL.baseName()), + new Metric(HostedNodeAdminMetrics.CPU_SYS_UTIL.baseName()), + new Metric(HostedNodeAdminMetrics.CPU_THROTTLED_TIME.baseName()), + new Metric(HostedNodeAdminMetrics.CPU_THROTTLED_CPU_TIME.baseName()), + new Metric(HostedNodeAdminMetrics.CPU_VCPUS.baseName()), + new Metric(HostedNodeAdminMetrics.DISK_LIMIT.baseName()), + new Metric(HostedNodeAdminMetrics.DISK_USED.baseName()), + new Metric(HostedNodeAdminMetrics.DISK_UTIL.baseName()), + new Metric(HostedNodeAdminMetrics.MEM_LIMIT.baseName()), + new Metric(HostedNodeAdminMetrics.MEM_USED.baseName()), + new Metric(HostedNodeAdminMetrics.MEM_UTIL.baseName()), + new Metric(HostedNodeAdminMetrics.MEM_TOTAL_USED.baseName()), + new Metric(HostedNodeAdminMetrics.MEM_TOTAL_UTIL.baseName()), + new Metric(HostedNodeAdminMetrics.GPU_UTIL.baseName()), + new Metric(HostedNodeAdminMetrics.GPU_MEM_USED.baseName()), + new Metric(HostedNodeAdminMetrics.GPU_MEM_TOTAL.baseName()) ); Set<Metric> nonDockerNodeMetrics = // Disk metrics should be based on /home, or else '/' - or simply add filesystem as dimension - ImmutableSet.of(new Metric("cpu.busy.pct", CPU_UTIL), - new Metric("mem.used.pct", MEM_UTIL), - new Metric("mem.active.kb", MEM_USED), - new Metric("mem.total.kb", MEM_LIMIT), - new Metric("used.kb", DISK_USED) + ImmutableSet.of(new Metric("cpu.busy.pct", HostedNodeAdminMetrics.CPU_UTIL.baseName()), + new Metric("mem.used.pct", HostedNodeAdminMetrics.MEM_UTIL.baseName()), + new Metric("mem.active.kb", HostedNodeAdminMetrics.MEM_USED.baseName()), + new Metric("mem.total.kb", HostedNodeAdminMetrics.MEM_LIMIT.baseName()), + new Metric("used.kb", HostedNodeAdminMetrics.DISK_USED.baseName()) ); Set<Metric> systemMetrics = ImmutableSet.<Metric>builder() diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 95ec1431cdd..5f8d8148b41 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -1,10 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; +import com.yahoo.metrics.ConfigServerMetrics; import com.yahoo.metrics.ContainerMetrics; import com.yahoo.metrics.DistributorMetrics; +import com.yahoo.metrics.LogdMetrics; +import com.yahoo.metrics.RoutingLayerMetrics; import com.yahoo.metrics.SearchNodeMetrics; +import com.yahoo.metrics.SentinelMetrics; +import com.yahoo.metrics.SlobrokMetrics; import com.yahoo.metrics.StorageMetrics; +import com.yahoo.metrics.NodeAdminMetrics; import com.yahoo.metrics.Suffix; import java.util.Collections; @@ -55,12 +61,10 @@ public class VespaMetricSet { private static Set<Metric> getSentinelMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); - addMetric(metrics, "sentinel.restarts.count"); - addMetric(metrics, "sentinel.totalRestarts.last"); - addMetric(metrics, "sentinel.uptime.last"); - - addMetric(metrics, "sentinel.running.count"); - addMetric(metrics, "sentinel.running.last"); + addMetric(metrics, SentinelMetrics.SENTINEL_RESTARTS.count()); + addMetric(metrics, SentinelMetrics.SENTINEL_TOTAL_RESTARTS.last()); + addMetric(metrics, SentinelMetrics.SENTINEL_UPTIME.last()); + addMetric(metrics, SentinelMetrics.SENTINEL_RUNNING, EnumSet.of(count, last)); return metrics; } @@ -68,39 +72,41 @@ public class VespaMetricSet { private static Set<Metric> getOtherMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); - addMetric(metrics, "slobrok.heartbeats.failed.count"); - addMetric(metrics, "slobrok.missing.consensus.count"); + addMetric(metrics, SlobrokMetrics.SLOBROK_HEARTBEATS_FAILED.count()); + addMetric(metrics, SlobrokMetrics.SLOBROK_MISSING_CONSENSUS.count()); - addMetric(metrics, "logd.processed.lines.count"); - addMetric(metrics, "worker.connections.max"); - addMetric(metrics, "endpoint.certificate.expiry.seconds"); + addMetric(metrics, LogdMetrics.LOGD_PROCESSED_LINES.count()); // Java (JRT) TLS metrics - addMetric(metrics, "jrt.transport.tls-certificate-verification-failures"); - addMetric(metrics, "jrt.transport.peer-authorization-failures"); - addMetric(metrics, "jrt.transport.server.tls-connections-established"); - addMetric(metrics, "jrt.transport.client.tls-connections-established"); - addMetric(metrics, "jrt.transport.server.unencrypted-connections-established"); - addMetric(metrics, "jrt.transport.client.unencrypted-connections-established"); + addMetric(metrics, ContainerMetrics.JRT_TRANSPORT_TLS_CERTIFICATE_VERIFICATION_FAILURES.baseName()); + addMetric(metrics, ContainerMetrics.JRT_TRANSPORT_PEER_AUTHORIZATION_FAILURES.baseName()); + addMetric(metrics, ContainerMetrics.JRT_TRANSPORT_SERVER_TLS_CONNECIONTS_ESTABLISHED.baseName()); + addMetric(metrics, ContainerMetrics.JRT_TRANSPORT_CLIENT_TLS_CONNECTIONS_ESTABLISHED.baseName()); + addMetric(metrics, ContainerMetrics.JRT_TRANSPORT_SERVER_UNENCRYPTED_CONNECTIONS_ESTABLISHED.baseName()); + addMetric(metrics, ContainerMetrics. JRT_TRANSPORT_CLIENT_UNENCRYPTED_CONNECTIONS_ESTABLISHED. baseName()); // C++ TLS metrics - addMetric(metrics, "vds.server.network.tls-handshakes-failed"); - addMetric(metrics, "vds.server.network.peer-authorization-failures"); - addMetric(metrics, "vds.server.network.client.tls-connections-established"); - addMetric(metrics, "vds.server.network.server.tls-connections-established"); - addMetric(metrics, "vds.server.network.client.insecure-connections-established"); - addMetric(metrics, "vds.server.network.server.insecure-connections-established"); - addMetric(metrics, "vds.server.network.tls-connections-broken"); - addMetric(metrics, "vds.server.network.failed-tls-config-reloads"); + addMetric(metrics, StorageMetrics.VDS_SERVER_NETWORK_TLS_HANDSHAKES_FAILED.count()); + addMetric(metrics, StorageMetrics.VDS_SERVER_NETWORK_PEER_AUTHORIZATION_FAILURES.count()); + addMetric(metrics, StorageMetrics.VDS_SERVER_NETWORK_CLIENT_TLS_CONNECTIONS_ESTABLISHED.count()); + addMetric(metrics, StorageMetrics.VDS_SERVER_NETWORK_SERVER_TLS_CONNECTIONS_ESTABLISHED.count()); + addMetric(metrics, StorageMetrics.VDS_SERVER_NETWORK_CLIENT_INSECURE_CONNECTIONS_ESTABLISHED.count()); + addMetric(metrics, StorageMetrics.VDS_SERVER_NETWORK_SERVER_INSECURE_CONNECTIONS_ESTABLISHED.count()); + addMetric(metrics, StorageMetrics.VDS_SERVER_NETWORK_TLS_CONNECTIONS_BROKEN.count()); + addMetric(metrics, StorageMetrics.VDS_SERVER_NETWORK_FAILED_TLS_CONFIG_RELOADS.count()); // C++ capability metrics - addMetric(metrics, "vds.server.network.rpc-capability-checks-failed"); - addMetric(metrics, "vds.server.network.status-capability-checks-failed"); + addMetric(metrics, StorageMetrics.VDS_SERVER_NETWORK_RPC_CAPABILITY_CHECKS_FAILED.count()); + addMetric(metrics, StorageMetrics.VDS_SERVER_NETWORK_STATUS_CAPABILITY_CHECKS_FAILED.count()); // C++ Fnet metrics - addMetric(metrics, "vds.server.fnet.num-connections"); + addMetric(metrics, StorageMetrics.VDS_SERVER_FNET_NUM_CONNECTIONS.count()); + + // NodeAdmin certificate + addMetric(metrics, NodeAdminMetrics.ENDPOINT_CERTIFICATE_EXPIRY_SECONDS.baseName()); + addMetric(metrics, NodeAdminMetrics.NODE_CERTIFICATE_EXPIRY_SECONDS.baseName()); - // Node certificate - addMetric(metrics, "node-certificate.expiry.seconds"); + // Routing layer metrics + addMetric(metrics, RoutingLayerMetrics.WORKER_CONNECTIONS.max()); // Hosted Vespa only (routing layer) return metrics; } @@ -108,22 +114,20 @@ public class VespaMetricSet { private static Set<Metric> getConfigServerMetrics() { Set<Metric> metrics =new LinkedHashSet<>(); - addMetric(metrics, "configserver.requests.count"); - addMetric(metrics, "configserver.failedRequests.count"); - addMetric(metrics, "configserver.latency.max"); - addMetric(metrics, "configserver.latency.sum"); - addMetric(metrics, "configserver.latency.count"); - addMetric(metrics, "configserver.cacheConfigElems.last"); - addMetric(metrics, "configserver.cacheChecksumElems.last"); - addMetric(metrics, "configserver.hosts.last"); - addMetric(metrics, "configserver.delayedResponses.count"); - addMetric(metrics, "configserver.sessionChangeErrors.count"); - - addMetric(metrics, "configserver.zkZNodes.last"); - addMetric(metrics, "configserver.zkAvgLatency.last"); - addMetric(metrics, "configserver.zkMaxLatency.last"); - addMetric(metrics, "configserver.zkConnections.last"); - addMetric(metrics, "configserver.zkOutstandingRequests.last"); + addMetric(metrics, ConfigServerMetrics.REQUESTS.count()); + addMetric(metrics, ConfigServerMetrics.FAILED_REQUESTS.count()); + addMetric(metrics, ConfigServerMetrics.LATENCY, EnumSet.of(max, sum, count)); + addMetric(metrics, ConfigServerMetrics.CACHE_CONFIG_ELEMS.last()); + addMetric(metrics, ConfigServerMetrics.CACHE_CHECKSUM_ELEMS.last()); + addMetric(metrics, ConfigServerMetrics.HOSTS.last()); + addMetric(metrics, ConfigServerMetrics.DELAYED_RESPONSES.count()); + addMetric(metrics, ConfigServerMetrics.SESSION_CHANGE_ERRORS.count()); + + addMetric(metrics, ConfigServerMetrics.ZK_Z_NODES.last()); + addMetric(metrics, ConfigServerMetrics.ZK_AVG_LATENCY.last()); + addMetric(metrics, ConfigServerMetrics.ZK_MAX_LATENCY.last()); + addMetric(metrics, ConfigServerMetrics.ZK_CONNECTIONS.last()); + addMetric(metrics, ConfigServerMetrics.ZK_OUTSTANDING_REQUESTS.last()); return metrics; } @@ -131,6 +135,8 @@ public class VespaMetricSet { private static Set<Metric> getContainerMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); + addMetric(metrics, ContainerMetrics.APPLICATION_GENERATION.baseName()); + addMetric(metrics, ContainerMetrics.HANDLED_REQUESTS.count()); addMetric(metrics, ContainerMetrics.HANDLED_LATENCY, EnumSet.of(sum, count, max)); @@ -337,6 +343,8 @@ public class VespaMetricSet { private static Set<Metric> getSearchNodeMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_CONFIG_GENERATION.last()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_TOTAL.last()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_READY.last()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_ACTIVE.last()); @@ -717,6 +725,18 @@ public class VespaMetricSet { private static void addMetric(Set<Metric> metrics, DistributorMetrics metric, EnumSet<Suffix> suffixes) { suffixes.forEach(suffix -> metrics.add(new Metric(metric.baseName() + "." + suffix.suffix()))); } + private static void addMetric(Set<Metric> metrics, SentinelMetrics metric, EnumSet<Suffix> suffixes) { + suffixes.forEach(suffix -> metrics.add(new Metric(metric.baseName() + "." + suffix.suffix()))); + } + private static void addMetric(Set<Metric> metrics, SlobrokMetrics metric, EnumSet<Suffix> suffixes) { + suffixes.forEach(suffix -> metrics.add(new Metric(metric.baseName() + "." + suffix.suffix()))); + } + private static void addMetric(Set<Metric> metrics, LogdMetrics metric, EnumSet<Suffix> suffixes) { + suffixes.forEach(suffix -> metrics.add(new Metric(metric.baseName() + "." + suffix.suffix()))); + } + private static void addMetric(Set<Metric> metrics, ConfigServerMetrics metric, EnumSet<Suffix> suffixes) { + suffixes.forEach(suffix -> metrics.add(new Metric(metric.baseName() + "." + suffix.suffix()))); + } private static void addMetric(Set<Metric> metrics, String metricName, Iterable<String> aggregateSuffices) { for (String suffix : aggregateSuffices) { metrics.add(new Metric(metricName + "." + suffix)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java index b9ed8a3c97c..36ebbe41637 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java @@ -38,7 +38,7 @@ public class UserConfigBuilder { ConfigDefinitionKey key = DomConfigPayloadBuilder.parseConfigName(element); Optional<ConfigDefinition> def = configDefinitionStore.getConfigDefinition(key); - if ( ! def.isPresent()) { // TODO: Fail instead of warn + if (def.isEmpty()) { // TODO: Fail instead of warn logger.logApplicationPackage(Level.WARNING, "Unable to find config definition '" + key.asFileName() + "'. Please ensure that the name is spelled correctly, and that the def file is included in a bundle."); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java index 3ad81ef9f57..c6844619457 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java @@ -1,12 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder; -import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.ConfigModelRepo; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; -import com.yahoo.config.model.ApplicationConfigProducerRoot; /** * Base class for classes capable of building vespa model. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 567ccbfa88b..80000e54b1b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -123,7 +123,8 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { ClusterSpec.Type.admin, ClusterSpec.Id.from(clusterId), context.getDeployLogger(), - false) + false, + context.clusterInfo().build()) .keySet(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java deleted file mode 100644 index 3491f219a8e..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.builder.xml.dom; - -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.producer.AnyConfigProducer; -import com.yahoo.config.model.producer.TreeConfigProducer; -import com.yahoo.text.XML; -import com.yahoo.vespa.model.container.ApplicationContainerCluster; -import com.yahoo.vespa.model.container.component.Handler; -import com.yahoo.vespa.model.container.component.UserBindingPattern; -import org.w3c.dom.Element; - -/** - * @author gjoranv - * @since 5.1.6 - */ -public class DomClientProviderBuilder extends DomHandlerBuilder { - - public DomClientProviderBuilder(ApplicationContainerCluster cluster) { - super(cluster); - } - - @Override - protected Handler doBuild(DeployState deployState, TreeConfigProducer<AnyConfigProducer> parent, Element clientElement) { - Handler client = createHandler(clientElement); - - for (Element binding : XML.getChildren(clientElement, "binding")) - client.addClientBindings(UserBindingPattern.fromPattern(XML.getValue(binding))); - - for (Element serverBinding : XML.getChildren(clientElement, "serverBinding")) - client.addServerBindings(UserBindingPattern.fromPattern(XML.getValue(serverBinding))); - - DomComponentBuilder.addChildren(deployState, parent, clientElement, client); - - return client; - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java index 5d17619b526..70bb80ec314 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.collections.Tuple2; -import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.FileReference; import com.yahoo.config.ModelReference; import com.yahoo.config.UrlReference; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java index 3cf8ec7375f..ed53a1d2267 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java @@ -38,7 +38,7 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilderB VIP_HANDLER_BINDING); private final ApplicationContainerCluster cluster; - private OptionalInt portBindingOverride; + private final OptionalInt portBindingOverride; public DomHandlerBuilder(ApplicationContainerCluster cluster) { this(cluster, OptionalInt.empty()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomRoutingBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomRoutingBuilder.java index 92b24f3f7ac..14a5b13d8d2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomRoutingBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomRoutingBuilder.java @@ -2,18 +2,20 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.config.application.Xml; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; -import com.yahoo.messagebus.routing.*; +import com.yahoo.messagebus.routing.ApplicationSpec; +import com.yahoo.messagebus.routing.HopSpec; +import com.yahoo.messagebus.routing.RouteSpec; +import com.yahoo.messagebus.routing.RoutingSpec; +import com.yahoo.messagebus.routing.RoutingTableSpec; import com.yahoo.text.XML; -import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.vespa.model.routing.Routing; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; - -import java.util.Arrays; import java.util.List; /** @@ -29,7 +31,7 @@ public class DomRoutingBuilder extends ConfigModelBuilder<Routing> { @Override public List<ConfigModelId> handlesElements() { - return Arrays.asList(ConfigModelId.fromName("routing")); + return List.of(ConfigModelId.fromName("routing")); } // Overrides ConfigModelBuilder. @@ -71,7 +73,7 @@ public class DomRoutingBuilder extends ConfigModelBuilder<Routing> { * @param element The element to base the route config on. */ private static void addRoutingTable(RoutingSpec routing, Element element) { - boolean verify = element.hasAttribute("verify") ? Boolean.valueOf(element.getAttribute("verify")) : true; + boolean verify = shouldVerify(element); RoutingTableSpec table = new RoutingTableSpec(element.getAttribute("protocol"), verify); NodeList children = element.getChildNodes(); @@ -94,7 +96,7 @@ public class DomRoutingBuilder extends ConfigModelBuilder<Routing> { * @return The corresponding route spec. */ private static RouteSpec createRouteSpec(Element element) { - boolean verify = element.hasAttribute("verify") ? Boolean.valueOf(element.getAttribute("verify")) : true; + boolean verify = shouldVerify(element); RouteSpec route = new RouteSpec(element.getAttribute("name"), verify); String hops = element.getAttribute("hops"); int from = 0; @@ -123,9 +125,9 @@ public class DomRoutingBuilder extends ConfigModelBuilder<Routing> { * @return The corresponding hop spec. */ private static HopSpec createHopSpec(Element element) { - boolean verify = element.hasAttribute("verify") ? Boolean.valueOf(element.getAttribute("verify")) : true; + boolean verify = shouldVerify(element); HopSpec hop = new HopSpec(element.getAttribute("name"), element.getAttribute("selector"), verify); - if (Boolean.valueOf(element.getAttribute("ignore-result"))) { + if (Boolean.parseBoolean(element.getAttribute("ignore-result"))) { hop.setIgnoreResult(true); } NodeList children = element.getElementsByTagName("recipient"); @@ -135,4 +137,8 @@ public class DomRoutingBuilder extends ConfigModelBuilder<Routing> { } return hop; } + + private static boolean shouldVerify(Element element) { + return !element.hasAttribute("verify") || Boolean.parseBoolean(element.getAttribute("verify")); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index b5fa451fa0b..c968e31325a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.collections.Pair; import com.yahoo.component.Version; @@ -266,8 +267,9 @@ public class NodesSpecification { ClusterSpec.Type clusterType, ClusterSpec.Id clusterId, DeployLogger logger, - boolean stateful) { - return provision(hostSystem, clusterType, clusterId, ZoneEndpoint.defaultEndpoint, logger, stateful); + boolean stateful, + ClusterInfo clusterInfo) { + return provision(hostSystem, clusterType, clusterId, ZoneEndpoint.defaultEndpoint, logger, stateful, clusterInfo); } public Map<HostResource, ClusterMembership> provision(HostSystem hostSystem, @@ -275,7 +277,8 @@ public class NodesSpecification { ClusterSpec.Id clusterId, ZoneEndpoint zoneEndpoint, DeployLogger logger, - boolean stateful) { + boolean stateful, + ClusterInfo info) { if (combinedId.isPresent()) clusterType = ClusterSpec.Type.combined; ClusterSpec cluster = ClusterSpec.request(clusterType, clusterId) @@ -286,7 +289,7 @@ public class NodesSpecification { .loadBalancerSettings(zoneEndpoint) .stateful(stateful) .build(); - return hostSystem.allocateHosts(cluster, Capacity.from(min, max, groupSize, required, canFail, cloudAccount), logger); + return hostSystem.allocateHosts(cluster, Capacity.from(min, max, groupSize, required, canFail, cloudAccount, info), logger); } private static Pair<NodeResources, NodeResources> nodeResources(ModelElement nodesElement) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java index 8398df6f5ac..07e879f0e9c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.builder.xml.dom; import ai.vespa.validation.Validation; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.ConfigModelRepo; import com.yahoo.config.model.builder.xml.XmlHelper; @@ -24,12 +23,8 @@ import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import com.yahoo.vespa.model.content.Content; import com.yahoo.vespa.model.search.SearchCluster; import org.w3c.dom.Element; - -import java.util.HashSet; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.Map; -import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Pattern; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index 49292bd6df7..0be3c825614 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -26,9 +26,9 @@ public class ContainerModelEvaluation implements OnnxModelsConfig.Producer, RankingExpressionsConfig.Producer { - private final static String EVALUATION_BUNDLE_NAME = "model-evaluation"; - private final static String INTEGRATION_BUNDLE_NAME = "model-integration"; - private final static String ONNXRUNTIME_BUNDLE_NAME = "container-onnxruntime.jar"; + public final static String EVALUATION_BUNDLE_NAME = "model-evaluation"; + public final static String INTEGRATION_BUNDLE_NAME = "model-integration"; + public final static String ONNXRUNTIME_BUNDLE_NAME = "container-onnxruntime.jar"; private final static String EVALUATOR_NAME = ModelsEvaluator.class.getName(); private final static String REST_HANDLER_NAME = "ai.vespa.models.handler.ModelsEvaluationHandler"; @@ -44,6 +44,7 @@ public class ContainerModelEvaluation implements public ContainerModelEvaluation(ApplicationContainerCluster cluster, RankProfileList rankProfileList) { this.rankProfileList = Objects.requireNonNull(rankProfileList, "rankProfileList cannot be null"); cluster.addSimpleComponent(EVALUATOR_NAME, null, EVALUATION_BUNDLE_NAME); + cluster.addSimpleComponent("ai.vespa.modelintegration.evaluator.OnnxRuntime", null, INTEGRATION_BUNDLE_NAME); cluster.addComponent(ContainerModelEvaluation.getHandler()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java index 29b1edc1397..19df9a4064f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java @@ -10,6 +10,10 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import static com.yahoo.vespa.model.container.ContainerModelEvaluation.EVALUATION_BUNDLE_NAME; +import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATION_BUNDLE_NAME; +import static com.yahoo.vespa.model.container.ContainerModelEvaluation.ONNXRUNTIME_BUNDLE_NAME; + /** * NOTE: Stable ordering of bundles in config is handled by {@link ContainerCluster#addPlatformBundle(Path)} * @@ -53,7 +57,10 @@ public class PlatformBundles { public static final Set<Path> SEARCH_AND_DOCPROC_BUNDLES = toBundlePaths( SEARCH_AND_DOCPROC_BUNDLE, "docprocs", - "linguistics-components" + "linguistics-components", + EVALUATION_BUNDLE_NAME, + INTEGRATION_BUNDLE_NAME, + ONNXRUNTIME_BUNDLE_NAME ); private static Set<Path> toBundlePaths(String... bundleNames) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 86c48407775..414d4c817c7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -1,12 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.search; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.container.QrSearchersConfig; import com.yahoo.prelude.semantics.SemanticRulesConfig; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.search.pagetemplates.PageTemplatesConfig; import com.yahoo.search.query.profile.config.QueryProfilesConfig; +import com.yahoo.search.ranking.RankProfilesEvaluatorFactory; import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; @@ -23,6 +25,7 @@ import java.util.Collection; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static com.yahoo.vespa.model.container.PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE; @@ -44,21 +47,32 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> private final ApplicationContainerCluster owningCluster; private final List<SearchCluster> searchClusters = new LinkedList<>(); + private final Collection<String> schemasWithGlobalPhase; + private final boolean globalPhase; private QueryProfiles queryProfiles; private SemanticRules semanticRules; private PageTemplates pageTemplates; - public ContainerSearch(ApplicationContainerCluster cluster, SearchChains chains) { + public ContainerSearch(DeployState deployState, ApplicationContainerCluster cluster, SearchChains chains) { super(chains); + this.globalPhase = deployState.featureFlags().enableGlobalPhase(); + this.schemasWithGlobalPhase = getSchemasWithGlobalPhase(deployState); this.owningCluster = cluster; owningCluster.addComponent(Component.fromClassAndBundle(CompiledQueryProfileRegistry.class, SEARCH_AND_DOCPROC_BUNDLE)); owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.schema.SchemaInfo.class, SEARCH_AND_DOCPROC_BUNDLE)); owningCluster.addComponent(Component.fromClassAndBundle(SearchStatusExtension.class, SEARCH_AND_DOCPROC_BUNDLE)); + owningCluster.addComponent(Component.fromClassAndBundle(RankProfilesEvaluatorFactory.class, SEARCH_AND_DOCPROC_BUNDLE)); + owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.ranking.GlobalPhaseRanker.class, SEARCH_AND_DOCPROC_BUNDLE)); cluster.addSearchAndDocprocBundles(); } + private static Collection<String> getSchemasWithGlobalPhase(DeployState state) { + return state.rankProfileRegistry().all().stream() + .filter(rp -> rp.getGlobalPhase() != null).map(rp -> rp.schema().getName()).collect(Collectors.toSet()); + } + public void connectSearchClusters(Map<String, SearchCluster> searchClusters) { this.searchClusters.addAll(searchClusters.values()); initializeDispatchers(searchClusters.values()); @@ -68,9 +82,19 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> /** Adds a Dispatcher component to the owning container cluster for each search cluster */ private void initializeDispatchers(Collection<SearchCluster> searchClusters) { for (SearchCluster searchCluster : searchClusters) { - if ( ! ( searchCluster instanceof IndexedSearchCluster)) continue; - var dispatcher = new DispatcherComponent((IndexedSearchCluster)searchCluster); - owningCluster.addComponent(dispatcher); + if (searchCluster instanceof IndexedSearchCluster indexed) { + var dispatcher = new DispatcherComponent(indexed); + owningCluster.addComponent(dispatcher); + } + if (globalPhase) { + for (var documentDb : searchCluster.getDocumentDbs()) { + if (!schemasWithGlobalPhase.contains(documentDb.getSchemaName())) continue; + var factory = new RankProfilesEvaluatorComponent(documentDb); + if (! owningCluster.getComponentsMap().containsKey(factory.getComponentId())) { + owningCluster.addComponent(factory); + } + } + } } } @@ -133,16 +157,17 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> public void getConfig(QrSearchersConfig.Builder builder) { for (int i = 0; i < searchClusters.size(); i++) { SearchCluster sys = findClusterWithId(searchClusters, i); - QrSearchersConfig.Searchcluster.Builder scB = new QrSearchersConfig.Searchcluster.Builder(). - name(sys.getClusterName()); - for (SchemaInfo spec : sys.schemas().values()) { - scB.searchdef(spec.fullSchema().getName()); - } - scB.rankprofiles(new QrSearchersConfig.Searchcluster.Rankprofiles.Builder().configid(sys.getConfigId())); - scB.indexingmode(QrSearchersConfig.Searchcluster.Indexingmode.Enum.valueOf(sys.getIndexingModeName())); - if ( ! (sys instanceof IndexedSearchCluster)) { + QrSearchersConfig.Searchcluster.Builder scB = new QrSearchersConfig.Searchcluster.Builder(). + name(sys.getClusterName()); + for (SchemaInfo spec : sys.schemas().values()) { + scB.searchdef(spec.fullSchema().getName()); + } + scB.rankprofiles(new QrSearchersConfig.Searchcluster.Rankprofiles.Builder().configid(sys.getConfigId())); + scB.indexingmode(QrSearchersConfig.Searchcluster.Indexingmode.Enum.valueOf(sys.getIndexingModeName())); + scB.globalphase(globalPhase); + if ( ! (sys instanceof IndexedSearchCluster)) { scB.storagecluster(new QrSearchersConfig.Searchcluster.Storagecluster.Builder(). - routespec(((StreamingSearchCluster)sys).getStorageRouteSpec())); + routespec(((StreamingSearchCluster)sys).getStorageRouteSpec())); } builder.searchcluster(scB); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankProfilesEvaluatorComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankProfilesEvaluatorComponent.java new file mode 100644 index 00000000000..75a2802ee53 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankProfilesEvaluatorComponent.java @@ -0,0 +1,49 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +import com.yahoo.config.model.producer.AnyConfigProducer; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.search.ranking.RankProfilesEvaluator; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; +import com.yahoo.vespa.model.container.ContainerModelEvaluation; +import com.yahoo.vespa.model.container.PlatformBundles; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.search.DocumentDatabase; + +public class RankProfilesEvaluatorComponent + extends Component<AnyConfigProducer, ComponentModel> + implements + RankProfilesConfig.Producer, + RankingConstantsConfig.Producer, + RankingExpressionsConfig.Producer, + OnnxModelsConfig.Producer +{ + private final DocumentDatabase ddb; + + public RankProfilesEvaluatorComponent(DocumentDatabase db) { + super(toComponentModel(db.getSchemaName())); + ddb = db; + } + + private static ComponentModel toComponentModel(String p) { + String myComponentId = "ranking-expression-evaluator." + p; + return new ComponentModel(myComponentId, + RankProfilesEvaluator.class.getName(), + PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE); + } + + @Override + public void getConfig(RankProfilesConfig.Builder builder) { ddb.getConfig(builder); } + + @Override + public void getConfig(RankingExpressionsConfig.Builder builder) { ddb.getConfig(builder); } + + @Override + public void getConfig(RankingConstantsConfig.Builder builder) { ddb.getConfig(builder); } + + @Override + public void getConfig(OnnxModelsConfig.Builder builder) { ddb.getConfig(builder); } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index ceba3864296..36d34b99223 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.Version; @@ -345,10 +346,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addDeploymentSpecConfig(ApplicationContainerCluster cluster, ConfigModelContext context, DeployLogger deployLogger) { if ( ! context.getDeployState().isHosted()) return; - Optional<DeploymentSpec> deploymentSpec = app.getDeployment().map(DeploymentSpec::fromXml); + DeploymentSpec deploymentSpec = app.getDeploymentSpec(); if (deploymentSpec.isEmpty()) return; - for (var deprecatedElement : deploymentSpec.get().deprecatedElements()) { + for (var deprecatedElement : deploymentSpec.deprecatedElements()) { deployLogger.logApplicationPackage(WARNING, deprecatedElement.humanReadableString()); } @@ -358,8 +359,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { context.getDeployState().getProperties().ztsUrl(), context.getDeployState().getProperties().athenzDnsSuffix(), context.getDeployState().zone(), - deploymentSpec.get()); - addRotationProperties(cluster, context.getDeployState().zone(), context.getDeployState().getEndpoints(), deploymentSpec.get()); + deploymentSpec); + addRotationProperties(cluster, context.getDeployState().zone(), context.getDeployState().getEndpoints(), deploymentSpec); } private void addRotationProperties(ApplicationContainerCluster cluster, Zone zone, Set<ContainerEndpoint> endpoints, DeploymentSpec spec) { @@ -732,7 +733,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { SearchChains searchChains = new DomSearchChainsBuilder() .build(deployState, containerCluster, producerSpec); - ContainerSearch containerSearch = new ContainerSearch(containerCluster, searchChains); + ContainerSearch containerSearch = new ContainerSearch(deployState, containerCluster, searchChains); applyApplicationPackageDirectoryConfigs(deployState.getApplicationPackage(), containerSearch); containerSearch.setQueryProfiles(deployState.getQueryProfiles()); @@ -863,10 +864,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { InstanceName instance = context.properties().applicationId().instance(); ZoneId zone = ZoneId.from(context.properties().zone().environment(), context.properties().zone().region()); - DeploymentSpec spec = context.getApplicationPackage().getDeployment() - .map(new DeploymentSpecXmlReader(false)::read) - .orElse(DeploymentSpec.empty); - return spec.zoneEndpoint(instance, zone, cluster); + return context.getApplicationPackage().getDeploymentSpec().zoneEndpoint(instance, zone, cluster); } private static Map<String, String> getEnvironmentVariables(Element environmentVariables) { @@ -924,22 +922,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { HostSystem hostSystem = cluster.hostSystem(); if (deployState.isHosted()) { // request just enough nodes to satisfy environment capacity requirement - ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, - ClusterSpec.Id.from(cluster.getName())) - .vespaVersion(deployState.getWantedNodeVespaVersion()) - .dockerImageRepository(deployState.getWantedDockerImageRepo()) - .build(); int nodeCount = deployState.zone().environment().isProduction() ? 2 : 1; - deployState.getDeployLogger().logApplicationPackage(Level.INFO, "Using " + nodeCount + - " nodes in " + cluster); - ClusterResources resources = new ClusterResources(nodeCount, 1, NodeResources.unspecified()); - Capacity capacity = Capacity.from(resources, - resources, - IntRange.empty(), - false, - !deployState.getProperties().isBootstrap(), - context.getDeployState().getProperties().cloudAccount()); - var hosts = hostSystem.allocateHosts(clusterSpec, capacity, log); + deployState.getDeployLogger().logApplicationPackage(Level.INFO, "Using " + nodeCount + " nodes in " + cluster); + var nodesSpec = NodesSpecification.dedicated(nodeCount, context); + var hosts = nodesSpec.provision(hostSystem, + ClusterSpec.Type.container, + ClusterSpec.Id.from(cluster.getName()), + deployState.getDeployLogger(), + false, + context.clusterInfo().build()); return createNodesFromHosts(hosts, cluster, context.getDeployState()); } else { @@ -956,15 +947,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private List<ApplicationContainer> createNodesFromNodeCount(ApplicationContainerCluster cluster, Element containerElement, Element nodesElement, ConfigModelContext context) { try { - NodesSpecification nodesSpecification = NodesSpecification.from(new ModelElement(nodesElement), context); - ClusterSpec.Id clusterId = ClusterSpec.Id.from(cluster.name()); - ZoneEndpoint zoneEndpoint = zoneEndpoint(context, clusterId); + var nodesSpecification = NodesSpecification.from(new ModelElement(nodesElement), context); + var clusterId = ClusterSpec.Id.from(cluster.name()); Map<HostResource, ClusterMembership> hosts = nodesSpecification.provision(cluster.getRoot().hostSystem(), ClusterSpec.Type.container, clusterId, - zoneEndpoint, + zoneEndpoint(context, clusterId), log, - getZooKeeper(containerElement) != null); + getZooKeeper(containerElement) != null, + context.clusterInfo().build()); return createNodesFromHosts(hosts, cluster, context.getDeployState()); } catch (IllegalArgumentException e) { @@ -998,7 +989,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { StorageGroup.provisionHosts(nodeSpecification, referenceId, cluster.getRoot().hostSystem(), - context.getDeployLogger()); + context); return createNodesFromHosts(hosts, cluster, context.getDeployState()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 71726b50391..a0240d28a3c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -426,6 +426,16 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> } builder.indexing.optimize(feedSequencerType); + setMaxFlushed(builder); + } + + private void setMaxFlushed(ProtonConfig.Builder builder) { + // maxflushed should be moved down to proton + double concurrency = builder.feeding.build().concurrency(); + if (concurrency > defaultFeedConcurrency) { + int maxFlushes = (int)Math.ceil(4 * concurrency); + builder.index.maxflushed(maxFlushes); + } } private boolean isGloballyDistributed(NewDocumentType docType) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index 31ec764fbde..52b2ce06dfe 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -187,10 +187,15 @@ public class StorageGroup { public static Map<HostResource, ClusterMembership> provisionHosts(NodesSpecification nodesSpecification, String clusterIdString, - HostSystem hostSystem, - DeployLogger logger) { + HostSystem hostSystem, + ConfigModelContext context) { ClusterSpec.Id clusterId = ClusterSpec.Id.from(clusterIdString); - return nodesSpecification.provision(hostSystem, ClusterSpec.Type.content, clusterId, logger, true); + return nodesSpecification.provision(hostSystem, + ClusterSpec.Type.content, + clusterId, + context.getDeployLogger(), + true, + context.clusterInfo().build()); } public static class Builder { @@ -203,7 +208,9 @@ public class StorageGroup { this.context = context; } - public StorageGroup buildRootGroup(DeployState deployState, RedundancyBuilder redundancyBuilder, ContentCluster owner) { + public StorageGroup buildRootGroup(DeployState deployState, + RedundancyBuilder redundancyBuilder, + ContentCluster owner) { try { if (owner.isHosted()) validateRedundancyAndGroups(deployState.zone().environment()); @@ -219,7 +226,7 @@ public class StorageGroup { GroupBuilder groupBuilder = collectGroup(owner.isHosted(), group, nodes, null, null); StorageGroup storageGroup = owner.isHosted() - ? groupBuilder.buildHosted(deployState, owner, Optional.empty()) + ? groupBuilder.buildHosted(deployState, owner, Optional.empty(), context) : groupBuilder.buildNonHosted(deployState, owner, Optional.empty()); Redundancy redundancy = redundancyBuilder.build(owner.isHosted(), storageGroup.subgroups.size(), @@ -334,12 +341,18 @@ public class StorageGroup { * @param parent the parent storage group, or empty if this is the root group * @return the storage group build by this */ - public StorageGroup buildHosted(DeployState deployState, ContentCluster owner, Optional<GroupBuilder> parent) { + public StorageGroup buildHosted(DeployState deployState, + ContentCluster owner, + Optional<GroupBuilder> parent, + ConfigModelContext context) { if (storageGroup.getIndex() != null) throw new IllegalArgumentException("Specifying individual groups is not supported on hosted applications"); Map<HostResource, ClusterMembership> hostMapping = nodeRequirement.isPresent() ? - provisionHosts(nodeRequirement.get(), owner.getStorageCluster().getClusterName(), owner.getRoot().hostSystem(), deployState.getDeployLogger()) : + provisionHosts(nodeRequirement.get(), + owner.getStorageCluster().getClusterName(), + owner.getRoot().hostSystem(), + context) : Collections.emptyMap(); Map<Optional<ClusterSpec.Group>, Map<HostResource, ClusterMembership>> hostGroups = collectAllocatedSubgroups(hostMapping); @@ -362,7 +375,7 @@ public class StorageGroup { storageGroup.nodes.add(createStorageNode(deployState, owner, host.getKey(), storageGroup, host.getValue())); } for (GroupBuilder subGroup : subGroups) { - storageGroup.subgroups.add(subGroup.buildHosted(deployState, owner, Optional.of(this))); + storageGroup.subgroups.add(subGroup.buildHosted(deployState, owner, Optional.of(this), context)); } } return storageGroup; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 137e19e7d86..7f4fc4cd89d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -150,7 +150,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem if (e != null) setupDocumentProcessing(c, e); } else if (c.persistenceFactory != null) { - throw new IllegalArgumentException("The specified content engine requires the <documents> element to be specified."); + throw new IllegalArgumentException("The <documents> element is mandatory in content cluster '" + clusterId + "'"); } ModelElement tuning = contentElement.child("tuning"); @@ -333,7 +333,8 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem ClusterSpec.Type.admin, ClusterSpec.Id.from(clusterName), context.getDeployLogger(), - true) + true, + context.clusterInfo().build()) .keySet(); admin.setClusterControllers(createClusterControllers(new ClusterControllerCluster(admin, "standalone", deployState), hosts, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 708a3f220ac..4ed8c5ab2e8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -203,9 +203,11 @@ public class IndexedSearchCluster extends SearchCluster } } + @Override public List<DocumentDatabase> getDocumentDbs() { return documentDbs; } + public boolean hasDocumentDB(String name) { for (DocumentDatabase db : documentDbs) { if (db.getName().equals(name)) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java index b6a7fb6182e..ee18eceb719 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java @@ -97,8 +97,10 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { if (usableMemoryGb() < MIN_MEMORY_PER_FLUSH_THREAD_GB) { builder.maxconcurrent(1); } + double min_concurrent_mem = usableMemoryGb() / (2*MIN_MEMORY_PER_FLUSH_THREAD_GB); + double min_concurrent_cpu = resources.vcpu() * MAX_FLUSH_THREAD_RATIO; builder.maxconcurrent(Math.min(builder.build().maxconcurrent(), - Math.max(1, (int)Math.ceil(resources.vcpu()*MAX_FLUSH_THREAD_RATIO)))); + (int)Math.ceil(Math.max(min_concurrent_mem, min_concurrent_cpu)))); } private void tuneFlushStrategyTlsSize(ProtonConfig.Flush.Memory.Builder builder) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index 61455fcbf34..818c43d0f93 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -55,6 +55,9 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer */ public abstract void deriveFromSchemas(DeployState deployState); + /** Returns the document databases contained in this cluster */ + public abstract List<DocumentDatabase> getDocumentDbs(); + /** Returns a list of the document type names used in this search cluster */ public List<String> getDocumentNames() { return schemas.values() diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index 6293a6d6cf2..3cedae2f896 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -17,6 +17,8 @@ import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; +import java.util.List; + /** * A search cluster of type streaming. * @@ -80,6 +82,15 @@ public class StreamingSearchCluster extends SearchCluster implements } @Override + public List<DocumentDatabase> getDocumentDbs() { + if (derived() == null) { + throw new IllegalArgumentException("missing derivedConfig"); + } + var db = new DocumentDatabase(this, docTypeName, derived()); + return List.of(db); + } + + @Override public void defaultDocumentsConfig() { } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/Duration.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/Duration.java index 5bb6d8cf6bf..f65b7a62f05 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/Duration.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/Duration.java @@ -20,8 +20,9 @@ import java.util.regex.Pattern; * Default is seconds. */ public class Duration { - private static Pattern pattern = Pattern.compile("([0-9\\.]+)\\s*([a-z]+)?"); - private static Map<String, Integer> unitMultiplier = new HashMap<>(); + + private static final Pattern pattern = Pattern.compile("([0-9\\.]+)\\s*([a-z]+)?"); + private static final Map<String, Integer> unitMultiplier = new HashMap<>(); static { unitMultiplier.put("s", 1000); unitMultiplier.put("d", 1000 * 3600 * 24); diff --git a/config-model/src/test/derived/globalphase_onnx_inside/.gitignore b/config-model/src/test/derived/globalphase_onnx_inside/.gitignore new file mode 100644 index 00000000000..d9609d7b326 --- /dev/null +++ b/config-model/src/test/derived/globalphase_onnx_inside/.gitignore @@ -0,0 +1 @@ +models.generated diff --git a/config-model/src/test/derived/globalphase_onnx_inside/files/ax_plus_b.onnx b/config-model/src/test/derived/globalphase_onnx_inside/files/ax_plus_b.onnx new file mode 100644 index 00000000000..17282d13dc3 --- /dev/null +++ b/config-model/src/test/derived/globalphase_onnx_inside/files/ax_plus_b.onnx @@ -0,0 +1,23 @@ +:© + +matrix_X +vector_AXA"MatMul + +XA +vector_Bvector_Y"AddlrZ +matrix_X + + +Z +vector_A + + +Z +vector_B + + +b +vector_Y + + +B
\ No newline at end of file diff --git a/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg b/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg new file mode 100644 index 00000000000..35bb1ccc3d2 --- /dev/null +++ b/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg @@ -0,0 +1,34 @@ +rankprofile[].name "default" +rankprofile[].fef.property[].name "rankingExpression(handicap).rankingScript" +rankprofile[].fef.property[].value "query(yy)" +rankprofile[].fef.property[].name "rankingExpression(handicap).type" +rankprofile[].fef.property[].value "tensor(d0[2])" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "rankingExpression(firstphase)" +rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" +rankprofile[].fef.property[].value "reduce(attribute(aa), sum)" +rankprofile[].fef.property[].name "vespa.rank.globalphase" +rankprofile[].fef.property[].value "rankingExpression(globalphase)" +rankprofile[].fef.property[].name "rankingExpression(globalphase).rankingScript" +rankprofile[].fef.property[].value "reduce(constant(ww) * (onnx(inside).foobar - rankingExpression(handicap)), sum)" +rankprofile[].fef.property[].name "vespa.match.feature" +rankprofile[].fef.property[].value "attribute(aa)" +rankprofile[].fef.property[].name "vespa.globalphase.rerankcount" +rankprofile[].fef.property[].value "13" +rankprofile[].fef.property[].name "vespa.type.attribute.aa" +rankprofile[].fef.property[].value "tensor(d1[3])" +rankprofile[].fef.property[].name "vespa.type.query.bb" +rankprofile[].fef.property[].value "tensor(d0[2])" +rankprofile[].fef.property[].name "vespa.type.query.yy" +rankprofile[].fef.property[].value "tensor(d0[2])" +rankprofile[].name "unranked" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "value(0)" +rankprofile[].fef.property[].name "vespa.hitcollector.heapsize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.hitcollector.arraysize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures" +rankprofile[].fef.property[].value "true" +rankprofile[].fef.property[].name "vespa.type.attribute.aa" +rankprofile[].fef.property[].value "tensor(d1[3])" diff --git a/config-model/src/test/derived/globalphase_onnx_inside/test.sd b/config-model/src/test/derived/globalphase_onnx_inside/test.sd new file mode 100644 index 00000000000..c38e318ce6b --- /dev/null +++ b/config-model/src/test/derived/globalphase_onnx_inside/test.sd @@ -0,0 +1,42 @@ +schema test { + + document test { + field aa type tensor(d1[3]) { + indexing: attribute + } + } + + constant xx { + file: files/const_xx.json + type: tensor(d0[2],d1[3]) + } + constant ww { + file: files/const_ww.json + type: tensor(d0[2]) + } + + rank-profile default { + inputs { + query(bb) tensor(d0[2]) + query(yy) tensor(d0[2]) + } + onnx-model inside { + file: files/ax_plus_b.onnx + input vector_A: attribute(aa) + input matrix_X: constant(xx) + input vector_B: query(bb) + output vector_Y: foobar + } + first-phase { + expression: sum(attribute(aa)) + } + function handicap() { + expression: query(yy) + } + global-phase { + rerank-count: 13 + expression: sum(constant(ww) * (onnx(inside).foobar - handicap)) + } + } + +} diff --git a/config-model/src/test/derived/rankingexpression/rank-profiles.cfg b/config-model/src/test/derived/rankingexpression/rank-profiles.cfg index ea8bc5f77e6..202669ae049 100644 --- a/config-model/src/test/derived/rankingexpression/rank-profiles.cfg +++ b/config-model/src/test/derived/rankingexpression/rank-profiles.cfg @@ -351,6 +351,8 @@ rankprofile[].fef.property[].name "rankingExpression(myplus).rankingScript" rankprofile[].fef.property[].value "attribute(foo1) + attribute(foo2)" rankprofile[].fef.property[].name "rankingExpression(mymul).rankingScript" rankprofile[].fef.property[].value "attribute(t1) * query(fromq)" +rankprofile[].fef.property[].name "rankingExpression(mymul).type" +rankprofile[].fef.property[].value "tensor(m{},v[3])" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "attribute(foo1)" rankprofile[].fef.property[].name "vespa.rank.secondphase" @@ -410,8 +412,6 @@ rankprofile[].fef.property[].value "rankingExpression(globalphase)" rankprofile[].fef.property[].name "rankingExpression(globalphase).rankingScript" rankprofile[].fef.property[].value "rankingExpression(myplus) + reduce(rankingExpression(mymul), sum) + firstPhase" rankprofile[].fef.property[].name "vespa.match.feature" -rankprofile[].fef.property[].value "query(fromq)" -rankprofile[].fef.property[].name "vespa.match.feature" rankprofile[].fef.property[].value "firstPhase" rankprofile[].fef.property[].name "vespa.match.feature" rankprofile[].fef.property[].value "attribute(t1)" diff --git a/config-model/src/test/derived/rankingmacros/rank-profiles.cfg b/config-model/src/test/derived/rankingmacros/rank-profiles.cfg new file mode 100644 index 00000000000..71b7bc7166c --- /dev/null +++ b/config-model/src/test/derived/rankingmacros/rank-profiles.cfg @@ -0,0 +1,83 @@ +rankprofile[].name "default" +rankprofile[].name "unranked" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "value(0)" +rankprofile[].fef.property[].name "vespa.hitcollector.heapsize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.hitcollector.arraysize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures" +rankprofile[].fef.property[].value "true" +rankprofile[].name "standalone" +rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" +rankprofile[].fef.property[].value "7 * attribute(num)" +rankprofile[].fef.property[].name "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86).rankingScript" +rankprofile[].fef.property[].value "4 * (match + match)" +rankprofile[].fef.property[].name "rankingExpression(macro_with_dollar$).rankingScript" +rankprofile[].fef.property[].value "69" +rankprofile[].fef.property[].name "rankingExpression(anotherfeature).rankingScript" +rankprofile[].fef.property[].value "10 * rankingExpression(myfeature)" +rankprofile[].fef.property[].name "rankingExpression(yetanotherfeature).rankingScript" +rankprofile[].fef.property[].value "100 * rankingExpression(myfeature)" +rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript" +rankprofile[].fef.property[].value "4 * (var1 + var2)" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "rankingExpression(firstphase)" +rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" +rankprofile[].fef.property[].value "match + fieldMatch(title) + rankingExpression(myfeature)" +rankprofile[].fef.property[].name "vespa.rank.secondphase" +rankprofile[].fef.property[].value "rankingExpression(secondphase)" +rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript" +rankprofile[].fef.property[].value "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86) + 0 * rankingExpression(macro_with_dollar$)" +rankprofile[].fef.property[].name "vespa.summary.feature" +rankprofile[].fef.property[].value "firstPhase" +rankprofile[].fef.property[].name "vespa.summary.feature" +rankprofile[].fef.property[].value "rankingExpression(myfeature)" +rankprofile[].fef.property[].name "vespa.summary.feature" +rankprofile[].fef.property[].value "rankingExpression(anotherfeature)" +rankprofile[].fef.property[].name "vespa.summary.feature" +rankprofile[].fef.property[].value "rankingExpression(yetanotherfeature)" +rankprofile[].fef.property[].name "vespa.summary.feature" +rankprofile[].fef.property[].value "rankingExpression(macro_with_dollar$)" +rankprofile[].fef.property[].name "vespa.feature.rename" +rankprofile[].fef.property[].value "rankingExpression(anotherfeature)" +rankprofile[].fef.property[].name "vespa.feature.rename" +rankprofile[].fef.property[].value "anotherfeature" +rankprofile[].fef.property[].name "vespa.feature.rename" +rankprofile[].fef.property[].value "rankingExpression(yetanotherfeature)" +rankprofile[].fef.property[].name "vespa.feature.rename" +rankprofile[].fef.property[].value "yetanotherfeature" +rankprofile[].fef.property[].name "vespa.feature.rename" +rankprofile[].fef.property[].value "rankingExpression(macro_with_dollar$)" +rankprofile[].fef.property[].name "vespa.feature.rename" +rankprofile[].fef.property[].value "macro_with_dollar$" +rankprofile[].name "constantsAndMacro" +rankprofile[].fef.property[].name "rankingExpression(c).rankingScript" +rankprofile[].fef.property[].value "attribute(num)" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "rankingExpression(firstphase)" +rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" +rankprofile[].fef.property[].value "attribute(num) * 2.0 + 3.0" +rankprofile[].fef.property[].name "vespa.summary.feature" +rankprofile[].fef.property[].value "firstPhase" +rankprofile[].name "doc" +rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" +rankprofile[].fef.property[].value "fieldMatch(title) + freshness(timestamp)" +rankprofile[].fef.property[].name "rankingExpression(otherfeature@6b0a229a66fcaa04).rankingScript" +rankprofile[].fef.property[].value "nativeRank(title,body)" +rankprofile[].fef.property[].name "rankingExpression(otherfeature).rankingScript" +rankprofile[].fef.property[].value "nativeRank(foo,body)" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "rankingExpression(firstphase)" +rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" +rankprofile[].fef.property[].value "rankingExpression(myfeature) * 10" +rankprofile[].fef.property[].name "vespa.rank.secondphase" +rankprofile[].fef.property[].value "rankingExpression(secondphase)" +rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript" +rankprofile[].fef.property[].value "rankingExpression(otherfeature@6b0a229a66fcaa04) * rankingExpression(myfeature)" +rankprofile[].fef.property[].name "vespa.summary.feature" +rankprofile[].fef.property[].value "rankingExpression(myfeature)" +rankprofile[].fef.property[].name "vespa.feature.rename" +rankprofile[].fef.property[].value "rankingExpression(myfeature)" +rankprofile[].fef.property[].name "vespa.feature.rename" +rankprofile[].fef.property[].value "myfeature" diff --git a/config-model/src/test/derived/rankingmacros/rankingmacros.sd b/config-model/src/test/derived/rankingmacros/rankingmacros.sd new file mode 100644 index 00000000000..84598cb483a --- /dev/null +++ b/config-model/src/test/derived/rankingmacros/rankingmacros.sd @@ -0,0 +1,105 @@ +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +schema rankingmacros { + + document rankingmacros { + field title type string { + indexing: index + } + field timestamp type long { + indexing: attribute + } + field description type string { + indexing: index + } + field num type int { + indexing: attribute + } + field abstract type string { + indexing: index + } + field body type string { + indexing: index + } + field usstaticrank type string { + indexing: attribute + } + field boostmax type string { + indexing: index + } + field entitytitle type string { + indexing: index + } + } + + rank-profile standalone { + macro fourtimessum(var1, var2) { + expression: 4*(var1+var2) + } + macro myfeature() { + expression { + 7 * attribute(num) + } + } + macro anotherfeature() { + expression: 10*myfeature + } + macro yetanotherfeature() { + expression: 100*rankingExpression(myfeature) # legacy form + } + macro macro_with_dollar$() { # Not allowed + expression: 69 + } + first-phase { + expression: match + fieldMatch(title) + myfeature + } + second-phase { + expression: fourtimessum(match,match) + 0 * macro_with_dollar$ + } + summary-features { + firstPhase + rankingExpression(myfeature) + anotherfeature + yetanotherfeature + macro_with_dollar$ + } + } + + # Profile with macro and constants + rank-profile constantsAndMacro { + macro c() { + expression: attribute(num) + } + + constants { + a: 2 + b: 3 + } + + first-phase { + expression: attribute(num) * a + b + } + + summary-features { + firstPhase + } + } + + # The example in the docs + rank-profile doc inherits default { + macro myfeature() { + expression: fieldMatch(title) + freshness(timestamp) + } + macro otherfeature(foo) { + expression{ nativeRank(foo, body) } + } + + first-phase { + expression: myfeature * 10 + } + second-phase { + expression: otherfeature(title) * myfeature + } + summary-features: myfeature + } + +} diff --git a/config-model/src/test/derived/renamedfeatures/rank-profiles.cfg b/config-model/src/test/derived/renamedfeatures/rank-profiles.cfg index d084401d920..ea2d051484b 100644 --- a/config-model/src/test/derived/renamedfeatures/rank-profiles.cfg +++ b/config-model/src/test/derived/renamedfeatures/rank-profiles.cfg @@ -56,6 +56,8 @@ rankprofile[].fef.property[].value "tensor(m{},v[3])" rankprofile[].name "withmf" rankprofile[].fef.property[].name "rankingExpression(mymul).rankingScript" rankprofile[].fef.property[].value "attribute(t1) * query(fromq)" +rankprofile[].fef.property[].name "rankingExpression(mymul).type" +rankprofile[].fef.property[].value "tensor(m{},v[3])" rankprofile[].fef.property[].name "rankingExpression(myplus).rankingScript" rankprofile[].fef.property[].value "attribute(foo1) + attribute(foo2)" rankprofile[].fef.property[].name "vespa.rank.firstphase" diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/HostsXmlProvisionerTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/HostsXmlProvisionerTest.java index 05b8681b5fa..d92fea24d51 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/HostsXmlProvisionerTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/HostsXmlProvisionerTest.java @@ -70,8 +70,6 @@ public class HostsXmlProvisionerTest { assertEquals(3, map.size()); assertCorrectNumberOfHosts(map, 3); assertTrue(map.keySet().containsAll(aliases)); - - assertEquals("", System.getProperty("zookeeper.vespa.clients")); } @Test diff --git a/config-model/src/test/java/com/yahoo/schema/derived/GlobalPhaseOnnxModelsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/GlobalPhaseOnnxModelsTestCase.java new file mode 100644 index 00000000000..2ff33dd70d8 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/GlobalPhaseOnnxModelsTestCase.java @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.jupiter.api.Test; +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Tests exporting with global-phase and ONNX models + * + * @author arnej + */ +public class GlobalPhaseOnnxModelsTestCase extends AbstractExportingTestCase { + + @Test + void testModelInRankProfile() throws IOException, ParseException { + assertCorrectDeriving("globalphase_onnx_inside"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/RankProfilesTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/RankProfilesTestCase.java index 5c24b32e275..a4c03291ce2 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/RankProfilesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/RankProfilesTestCase.java @@ -17,4 +17,9 @@ public class RankProfilesTestCase extends AbstractExportingTestCase { void testRankProfiles() throws IOException, ParseException { assertCorrectDeriving("rankprofiles", null, new TestProperties(), new TestableDeployLogger()); } + + @Test + void testMacrosInRankProfiles() throws IOException, ParseException { + assertCorrectDeriving("rankingmacros", null, new TestProperties(), new TestableDeployLogger()); + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java new file mode 100644 index 00000000000..0abb153696c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java @@ -0,0 +1,85 @@ +package com.yahoo.vespa.model; + +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.provision.InMemoryProvisioner; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author bratseth + */ +public class ClusterInfoTest { + + @Test + void bcp_deadline_is_passed_in_cluster_info() throws Exception { + var servicesXml = """ + <services version='1.0'> + <container id='testcontainer' version='1.0'> + <nodes count='3'/> + </container> + <content id='testcontent' version='1.0'> + <redundancy>2</redundancy> + <documents/> + </content> + </services> + """; + + var deploymentXml = """ + <deployment version='1.0'> + <prod> + <region>us-west-1</region> + <region>us-east-1</region> + </prod> + <bcp> + <group deadline='30m'> + <region fraction='0.5'>us-east-1</region> + <region>us-west-1</region> + </group> + <group> + <region fraction='0.5'>us-east-1</region> + </group> + </bcp> + </deployment> + """; + + var requestedInUsEast1 = requestedCapacityIn("us-east-1", servicesXml, deploymentXml); + assertEquals(Duration.ofMinutes(0), requestedInUsEast1.get(new ClusterSpec.Id("testcontainer")).clusterInfo().bcpDeadline()); + assertEquals(Duration.ofMinutes(0), requestedInUsEast1.get(new ClusterSpec.Id("testcontent")).clusterInfo().bcpDeadline()); + + var requestedInUsWest1 = requestedCapacityIn("us-west-1", servicesXml, deploymentXml); + assertEquals(Duration.ofMinutes(30), requestedInUsWest1.get(new ClusterSpec.Id("testcontainer")).clusterInfo().bcpDeadline()); + assertEquals(Duration.ofMinutes(30), requestedInUsWest1.get(new ClusterSpec.Id("testcontent")).clusterInfo().bcpDeadline()); + } + + private Map<ClusterSpec.Id, Capacity> requestedCapacityIn(String region, String servicesXml, String deploymentXml) throws Exception { + var applicationPackage = new MockApplicationPackage.Builder() + .withServices(servicesXml) + .withDeploymentSpec(deploymentXml) + .build(); + + var provisioner = new InMemoryProvisioner(10, true); + var deployState = new DeployState.Builder() + .applicationPackage(applicationPackage) + .zone(new Zone(Environment.prod, RegionName.from(region))) + .properties(new TestProperties().setHostedVespa(true) + .setZone(new Zone(Environment.prod, RegionName.from(region)))) + .modelHostProvisioner(provisioner) + .provisioned(provisioner.provisioned()) + .build(); + new VespaModel(new NullConfigModelRegistry(), deployState); + return deployState.provisioned().all(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/HostPortsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/HostPortsTest.java index a138ef71b2f..4a3cae37570 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/HostPortsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/HostPortsTest.java @@ -30,7 +30,7 @@ public class HostPortsTest { void next_available_baseport_is_BASE_PORT_plus_one_when_one_port_has_been_reserved() { HostPorts host = new HostPorts("myhostname"); MockRoot root = new MockRoot(); - host.reservePort(new TestService(root, 1), HostPorts.BASE_PORT, "foo"); + host.reservePort(new TestService(root, 1), HostPorts.BASE_PORT); assertThat(host.nextAvailableBaseport(1), is(HostPorts.BASE_PORT + 1)); } @@ -40,12 +40,12 @@ public class HostPortsTest { MockRoot root = new MockRoot(); for (int p = HostPorts.BASE_PORT; p < HostPorts.BASE_PORT + HostPorts.MAX_PORTS; p += 2) { - host.reservePort(new TestService(root, 1), p, "foo"); + host.reservePort(new TestService(root, 1), p); } assertThat(host.nextAvailableBaseport(2), is(0)); try { - host.reservePort(new TestService(root, 2), HostPorts.BASE_PORT, "bar"); + host.reservePort(new TestService(root, 2), HostPorts.BASE_PORT); } catch (RuntimeException e) { assertThat(e.getMessage(), containsString("Too many ports are reserved")); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CloudAccountChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CloudAccountChangeValidatorTest.java index fcc8c82a6e9..a8a063cb5fb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CloudAccountChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CloudAccountChangeValidatorTest.java @@ -1,5 +1,6 @@ package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.deploy.DeployState; @@ -57,7 +58,8 @@ class CloudAccountChangeValidatorTest { IntRange.empty(), false, false, - Optional.of(cloudAccount).filter(account -> !account.isUnspecified())); + Optional.of(cloudAccount).filter(account -> !account.isUnspecified()), + ClusterInfo.empty()); } private static VespaModel model(Provisioned provisioned) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 6ac4dbd17e3..cad36f5574c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -835,7 +835,7 @@ public class ContentBuilderTest extends DomBuilderTest { " </group>" + "</content>"); }); - assertTrue(exception.getMessage().contains("The specified content engine requires the <documents> element to be specified.")); + assertEquals("The <documents> element is mandatory in content cluster 'a'", exception.getMessage()); } private ProtonConfig getProtonConfig(ContentCluster content) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index ce2d5ca2da5..5973ef56962 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -485,7 +485,7 @@ public class ContainerClusterTest { if (isCombinedCluster) cluster.setHostClusterId("test-content-cluster"); cluster.setMemoryPercentage(memoryPercentage); - cluster.setSearch(new ContainerSearch(cluster, new SearchChains(cluster, "search-chain"))); + cluster.setSearch(new ContainerSearch(root.getDeployState(), cluster, new SearchChains(cluster, "search-chain"))); return cluster; } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java index 063f8f3109e..5b6c7b97875 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.ml; -import ai.vespa.modelintegration.evaluator.OnnxEvaluator; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import ai.vespa.models.evaluation.FunctionEvaluator; import ai.vespa.models.evaluation.ModelsEvaluator; import com.yahoo.tensor.Tensor; @@ -21,7 +21,7 @@ public class ModelsEvaluatorTest { void testModelsEvaluator() { // Assumption fails but test passes on Intel macs // Assumption fails and test fails on ARM64 - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); ModelsEvaluator modelsEvaluator = ModelsEvaluatorTester.create("src/test/cfg/application/stateless_eval"); assertEquals(3, modelsEvaluator.models().size()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java index 81c7b778ed3..b2ae4e553f5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java @@ -9,8 +9,10 @@ import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; +import com.yahoo.vespa.model.search.DocumentDatabase; import com.yahoo.vespa.model.search.SearchCluster; import java.util.HashMap; +import java.util.List; import java.util.Map; public class MockSearchClusters { @@ -28,6 +30,11 @@ public class MockSearchClusters { public void deriveFromSchemas(DeployState deployState) { } @Override + public List<DocumentDatabase> getDocumentDbs() { + return List.of(); + } + + @Override public int getRowBits() { return 0; } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java index caf0d22d44e..fc70a65b394 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.ml; -import ai.vespa.modelintegration.evaluator.OnnxEvaluator; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import ai.vespa.models.evaluation.Model; import ai.vespa.models.evaluation.ModelsEvaluator; import ai.vespa.models.handler.ModelsEvaluationHandler; @@ -27,7 +27,10 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeTrue; /** @@ -60,7 +63,7 @@ public class ModelEvaluationTest { @Test void testMl_serving() throws IOException { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); Path appDir = Path.fromString("src/test/cfg/application/ml_serving"); Path storedAppDir = appDir.append("copy"); try { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java index a731e9c7ccc..b0fe2c09227 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.ml; -import ai.vespa.modelintegration.evaluator.OnnxEvaluator; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import ai.vespa.models.evaluation.FunctionEvaluator; import ai.vespa.models.evaluation.Model; import ai.vespa.models.evaluation.ModelsEvaluator; @@ -45,7 +45,7 @@ public class StatelessOnnxEvaluationTest { @Test void testStatelessOnnxModelNameCollision() { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); Path appDir = Path.fromString("src/test/cfg/application/onnx_name_collision"); try { ImportedModelTester tester = new ImportedModelTester("onnx", appDir); @@ -66,7 +66,7 @@ public class StatelessOnnxEvaluationTest { @Test void testStatelessOnnxModelEvaluation() throws Exception { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); Path appDir = Path.fromString("src/test/cfg/application/onnx"); Path storedAppDir = appDir.append("copy"); try { @@ -91,7 +91,7 @@ public class StatelessOnnxEvaluationTest { @Test void testStatelessOnnxModelEvaluationWithGpu() { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); NodeResources resources = new NodeResources(4, 16, 125, 10, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local, NodeResources.Architecture.x86_64, diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java index a6d44d46dcb..9fe38512fc0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java @@ -183,6 +183,10 @@ public class NodeResourcesTuningTest { @Test public void require_that_concurrent_flush_threads_is_1_with_low_memory() { assertEquals(2, fromMemAndCpu(17, 9).flush().maxconcurrent()); + assertEquals(2, fromMemAndCpu(17, 64).flush().maxconcurrent()); // still capped by max + assertEquals(2, fromMemAndCpu(65, 8).flush().maxconcurrent()); // still capped by max + assertEquals(2, fromMemAndCpu(33, 8).flush().maxconcurrent()); + assertEquals(1, fromMemAndCpu(31, 8).flush().maxconcurrent()); assertEquals(1, fromMemAndCpu(15, 8).flush().maxconcurrent()); assertEquals(1, fromMemAndCpu(17, 8).flush().maxconcurrent()); assertEquals(1, fromMemAndCpu(15, 8).flush().maxconcurrent()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index ee885cdf43e..1df297c2bfc 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -87,7 +87,7 @@ public class DocumentDatabaseTestCase { verifyConcurrency(nameAndModes, xmlTuning, expectedConcurrency, null); } - private void verifyConcurrency(List<DocType> nameAndModes, String xmlTuning, double expectedConcurrency, Double featureFlagConcurrency) { + private ProtonConfig getConfig(List<DocType> nameAndModes, String xmlTuning, Double featureFlagConcurrency) { TestProperties properties = new TestProperties(); if (featureFlagConcurrency != null) { properties.setFeedConcurrency(featureFlagConcurrency); @@ -95,10 +95,30 @@ public class DocumentDatabaseTestCase { var tester = new SchemaTester(); VespaModel model = tester.createModel(nameAndModes, xmlTuning, new DeployState.Builder().properties(properties)); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - ProtonConfig proton = tester.getProtonConfig(contentSearchCluster); + return tester.getProtonConfig(contentSearchCluster); + } + + private void verifyConcurrency(List<DocType> nameAndModes, String xmlTuning, double expectedConcurrency, Double featureFlagConcurrency) { + ProtonConfig proton = getConfig(nameAndModes, xmlTuning, featureFlagConcurrency); assertEquals(expectedConcurrency, proton.feeding().concurrency(), SMALL); } + private void verifyMaxflushedFollowsConcurrency(double concurrency, int maxFlushed) { + String feedTuning = "<feeding> <concurrency>" + concurrency +"</concurrency>" + "</feeding>\n"; + ProtonConfig proton = getConfig(List.of(DocType.create("a", "index")), feedTuning, null); + assertEquals(maxFlushed, proton.index().maxflushed()); + } + + @Test + public void verifyThatMaxFlushedFollowsConcurrency() { + verifyMaxflushedFollowsConcurrency(0.1, 2); + verifyMaxflushedFollowsConcurrency(0.50, 2); + verifyMaxflushedFollowsConcurrency(0.51, 3); + verifyMaxflushedFollowsConcurrency(0.75, 3); + verifyMaxflushedFollowsConcurrency(0.76, 4); + verifyMaxflushedFollowsConcurrency(1.0, 4); + } + private void verifyFeedNiceness(List<DocType> nameAndModes, Double expectedNiceness, Double featureFlagNiceness) { TestProperties properties = new TestProperties(); if (featureFlagNiceness != null) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java index 2477d19d46c..c92715e5d4f 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java @@ -20,6 +20,7 @@ public final class Capacity { private final boolean canFail; private final NodeType type; private final Optional<CloudAccount> cloudAccount; + private final ClusterInfo clusterInfo; private Capacity(ClusterResources min, ClusterResources max, @@ -27,7 +28,8 @@ public final class Capacity { boolean required, boolean canFail, NodeType type, - Optional<CloudAccount> cloudAccount) { + Optional<CloudAccount> cloudAccount, + ClusterInfo clusterInfo) { validate(min); validate(max); if (max.smallerThan(min)) @@ -42,6 +44,7 @@ public final class Capacity { this.canFail = canFail; this.type = type; this.cloudAccount = Objects.requireNonNull(cloudAccount); + this.clusterInfo = clusterInfo; } private static void validate(ClusterResources resources) { @@ -77,12 +80,14 @@ public final class Capacity { return cloudAccount; } + public ClusterInfo clusterInfo() { return clusterInfo; } + public Capacity withLimits(ClusterResources min, ClusterResources max) { return withLimits(min, max, IntRange.empty()); } public Capacity withLimits(ClusterResources min, ClusterResources max, IntRange groupSize) { - return new Capacity(min, max, groupSize, required, canFail, type, cloudAccount); + return new Capacity(min, max, groupSize, required, canFail, type, cloudAccount, clusterInfo); } @Override @@ -98,25 +103,31 @@ public final class Capacity { /** Create a non-required, failable capacity request */ public static Capacity from(ClusterResources min, ClusterResources max) { - return from(min, max, IntRange.empty(), false, true, Optional.empty()); + return from(min, max, IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty()); } public static Capacity from(ClusterResources resources, boolean required, boolean canFail) { return from(resources, required, canFail, NodeType.tenant); } - // TODO: Remove after February 2023 + // TODO: Remove after March 2023 + public static Capacity from(ClusterResources min, ClusterResources max, IntRange groupSize, boolean required, boolean canFail, Optional<CloudAccount> cloudAccount) { + return new Capacity(min, max, groupSize, required, canFail, NodeType.tenant, cloudAccount, ClusterInfo.empty()); + } + + // TODO: Remove after March 2023 public static Capacity from(ClusterResources min, ClusterResources max, boolean required, boolean canFail) { - return new Capacity(min, max, IntRange.empty(), required, canFail, NodeType.tenant, Optional.empty()); + return new Capacity(min, max, IntRange.empty(), required, canFail, NodeType.tenant, Optional.empty(), ClusterInfo.empty()); } - // TODO: Remove after February 2023 + // TODO: Remove after March 2023 public static Capacity from(ClusterResources min, ClusterResources max, boolean required, boolean canFail, Optional<CloudAccount> cloudAccount) { - return new Capacity(min, max, IntRange.empty(), required, canFail, NodeType.tenant, cloudAccount); + return new Capacity(min, max, IntRange.empty(), required, canFail, NodeType.tenant, cloudAccount, ClusterInfo.empty()); } - public static Capacity from(ClusterResources min, ClusterResources max, IntRange groupSize, boolean required, boolean canFail, Optional<CloudAccount> cloudAccount) { - return new Capacity(min, max, groupSize, required, canFail, NodeType.tenant, cloudAccount); + public static Capacity from(ClusterResources min, ClusterResources max, IntRange groupSize, boolean required, boolean canFail, + Optional<CloudAccount> cloudAccount, ClusterInfo clusterInfo) { + return new Capacity(min, max, groupSize, required, canFail, NodeType.tenant, cloudAccount, clusterInfo); } /** Creates this from a node type */ @@ -125,7 +136,7 @@ public final class Capacity { } private static Capacity from(ClusterResources resources, boolean required, boolean canFail, NodeType type) { - return new Capacity(resources, resources, IntRange.empty(), required, canFail, type, Optional.empty()); + return new Capacity(resources, resources, IntRange.empty(), required, canFail, type, Optional.empty(), ClusterInfo.empty()); } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java new file mode 100644 index 00000000000..fe8acb0c3c0 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java @@ -0,0 +1,61 @@ +package com.yahoo.config.provision; + +import java.time.Duration; +import java.util.Objects; + +/** + * Auxiliary information about a cluster, provided by the config model to the node repo during a + * capacity request. + * + * @author bratseth + */ +public class ClusterInfo { + + private static final ClusterInfo empty = new ClusterInfo.Builder().build(); + + private final Duration bcpDeadline; + + private ClusterInfo(Builder builder) { + this.bcpDeadline = builder.bcpDeadline; + } + + public Duration bcpDeadline() { return bcpDeadline; } + + public static ClusterInfo empty() { return empty; } + + public boolean isEmpty() { return this.equals(empty); } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof ClusterInfo other)) return false; + if ( ! other.bcpDeadline.equals(this.bcpDeadline)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(bcpDeadline); + } + + @Override + public String toString() { + return "cluster info: [bcp deadline: " + bcpDeadline + "]"; + } + + public static class Builder { + + private Duration bcpDeadline = Duration.ofMinutes(0); + + public Builder bcpDeadline(Duration duration) { + this.bcpDeadline = Objects.requireNonNull(duration); + return this; + } + + public ClusterInfo build() { + return new ClusterInfo(this); + } + + } + +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index 8b2bf9fcbcc..8004d4dc951 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -15,7 +15,7 @@ public class NodeResources { private static final double cpuUnitCost = 0.11; private static final double memoryUnitCost = 0.011; private static final double diskUnitCost = 0.0004; - private static final double gpuUnitCost = 0; // TODO(mpolden): Decide cost of this + private static final double gpuUnitCost = 0.075; private static final NodeResources zero = new NodeResources(0, 0, 0, 0); private static final NodeResources unspecified = new NodeResources(0, 0, 0, 0); @@ -129,10 +129,6 @@ public class NodeResources { validate(memoryGb, "memory"); } - private double totalMemory() { - return count * memoryGb; - } - private boolean lessThan(GpuResources other) { return this.count < other.count || this.memoryGb < other.memoryGb; @@ -219,7 +215,7 @@ public class NodeResources { return (vcpu * cpuUnitCost) + (memoryGb * memoryUnitCost) + (diskGb * diskUnitCost) + - (gpuResources.totalMemory() * gpuUnitCost); + (gpuResources.count * gpuResources.memoryGb * gpuUnitCost); } public NodeResources withVcpu(double vcpu) { diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/CapacityTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/CapacityTest.java index 89c0e98b076..a7614bbc016 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/CapacityTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/CapacityTest.java @@ -21,7 +21,8 @@ public class CapacityTest { IntRange.empty(), false, true, - Optional.empty()); + Optional.empty(), + ClusterInfo.empty()); assertValidationFailure(new ClusterResources(4, 2, new NodeResources(1, 2, 3, 4)), new ClusterResources(2, 2, new NodeResources(1, 2, 3, 4))); assertValidationFailure(new ClusterResources(4, 4, new NodeResources(1, 2, 3, 4)), @@ -41,7 +42,7 @@ public class CapacityTest { private void assertValidationFailure(ClusterResources min, ClusterResources max) { try { - Capacity.from(min, max, IntRange.empty(), false, true, Optional.empty()); + Capacity.from(min, max, IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty()); fail("Expected exception with min " + min + " and max " + max); } catch (IllegalArgumentException e) { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java index cae70b41c8c..380e7552cc7 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java @@ -53,7 +53,7 @@ public class DelayedResponseHandler implements Runnable { responseHandler.returnOkResponse(request, config.get()); sentResponses.incrementAndGet(); } else { - log.log(Level.WARNING, "Timed out (timeout " + request.getTimeout() + ") getting config " + + log.log(Level.INFO, "Timed out (timeout " + request.getTimeout() + ") getting config " + request.getConfigKey() + ", will retry"); } } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java index 08e5c22c4e1..ab218174090 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java @@ -84,14 +84,14 @@ class RpcConfigSourceClient implements ConfigSourceClient, Runnable { for (String configSource : configSourceSet.getSources()) { Spec spec = new Spec(configSource); Target target = supervisor.connect(spec); - target.invokeSync(req, Duration.ofSeconds(30)); + target.invokeSync(req, Duration.ofSeconds(60)); if (target.isValid()) return; - log.log(Level.INFO, "Could not connect to config source at " + spec.toString()); + log.log(Level.INFO, "Could not connect to config source at " + spec); target.close(); } - log.log(Level.INFO, "Could not connect to any config source in set " + configSourceSet.toString() + + log.log(Level.INFO, "Could not connect to any config source in set " + configSourceSet + ", please make sure config server(s) are running."); } @@ -153,7 +153,7 @@ class RpcConfigSourceClient implements ConfigSourceClient, Runnable { subscriber.subscribe(); subscribers.put(configCacheKey, subscriber); } catch (ConfigurationRuntimeException e) { - log.log(Level.INFO, "Subscribe for '" + configCacheKey + "' failed, closing subscriber", e); + log.log(Level.INFO, "Subscribe for '" + configCacheKey + "' failed, closing subscriber: " + e.getMessage()); subscriber.cancel(); } } diff --git a/config/CMakeLists.txt b/config/CMakeLists.txt index 1c04aa3eaa8..561ddbc078c 100644 --- a/config/CMakeLists.txt +++ b/config/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalib vespalog fnet diff --git a/config/src/apps/vespa-configproxy-cmd/proxycmd.cpp b/config/src/apps/vespa-configproxy-cmd/proxycmd.cpp index 1d875e0ade2..2bc1c3e94d1 100644 --- a/config/src/apps/vespa-configproxy-cmd/proxycmd.cpp +++ b/config/src/apps/vespa-configproxy-cmd/proxycmd.cpp @@ -39,11 +39,11 @@ void ProxyCmd::invokeRPC() { void ProxyCmd::finiRPC() { if (_req != nullptr) { - _req->SubRef(); + _req->internal_subref(); _req = nullptr; } if (_target != nullptr) { - _target->SubRef(); + _target->internal_subref(); _target = NULL; } _server.reset(); diff --git a/config/src/apps/vespa-get-config/getconfig.cpp b/config/src/apps/vespa-get-config/getconfig.cpp index 74380390095..a31caefe054 100644 --- a/config/src/apps/vespa-get-config/getconfig.cpp +++ b/config/src/apps/vespa-get-config/getconfig.cpp @@ -12,6 +12,7 @@ #include <vespa/config/common/configresponse.h> #include <vespa/config/common/trace.h> #include <vespa/vespalib/util/signalhandler.h> +#include <cinttypes> #include <unistd.h> #include <sstream> @@ -84,7 +85,7 @@ void GetConfig::finiRPC() { if (_target != nullptr) { - _target->SubRef(); + _target->internal_subref(); _target = nullptr; } _server.reset(); diff --git a/config/src/apps/vespa-ping-configproxy/pingproxy.cpp b/config/src/apps/vespa-ping-configproxy/pingproxy.cpp index e8423eba233..83a10416852 100644 --- a/config/src/apps/vespa-ping-configproxy/pingproxy.cpp +++ b/config/src/apps/vespa-ping-configproxy/pingproxy.cpp @@ -59,7 +59,7 @@ void PingProxy::finiRPC() { if (_target != nullptr) { - _target->SubRef(); + _target->internal_subref(); _target = nullptr; } _server.reset(); @@ -146,7 +146,7 @@ PingProxy::main(int argc, char **argv) retval = 1; } } - req->SubRef(); + req->internal_subref(); finiRPC(); return retval; } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java index b4a2a29102b..4b72d8962bc 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java @@ -26,6 +26,7 @@ import java.util.logging.Logger; import static com.yahoo.jrt.ErrorCode.CONNECTION; import static java.util.logging.Level.FINE; import static java.util.logging.Level.FINEST; +import static java.util.logging.Level.INFO; import static java.util.logging.Level.SEVERE; import static java.util.logging.Level.WARNING; @@ -149,6 +150,9 @@ public class JRTConfigRequester implements RequestWaiter { if (jrtReq.errorCode() == CONNECTION) { log.log(FINE, () -> "Request failed: " + jrtReq.errorMessage() + "\nConnection spec: " + connection); + } else if (jrtReq.errorCode() == ErrorCode.APPLICATION_NOT_LOADED) { + log.log(INFO, () -> "Request failed: " + jrtReq.errorMessage() + + "\nConnection spec: " + connection); } else if (timeForLastLogWarning.isBefore(Instant.now().minus(delayBetweenWarnings))) { log.log(WARNING, "Request failed: " + ErrorCode.getName(jrtReq.errorCode()) + ". Connection spec: " + connection.getAddress() + diff --git a/config/src/tests/file_acquirer/file_acquirer_test.cpp b/config/src/tests/file_acquirer/file_acquirer_test.cpp index 8449a33a782..7ea6556c074 100644 --- a/config/src/tests/file_acquirer/file_acquirer_test.cpp +++ b/config/src/tests/file_acquirer/file_acquirer_test.cpp @@ -4,14 +4,12 @@ #include <vespa/fnet/frt/supervisor.h> #include <vespa/fnet/frt/rpcrequest.h> #include <vespa/fnet/transport.h> -#include <vespa/fastos/thread.h> #include <vespa/vespalib/util/stringfmt.h> using namespace config; struct ServerFixture : FRT_Invokable { fnet::frt::StandaloneFRT server; - FastOS_ThreadPool threadPool; FNET_Transport transport; FRT_Supervisor &orb; vespalib::string spec; @@ -26,14 +24,13 @@ struct ServerFixture : FRT_Invokable { ServerFixture() : server(), - threadPool(), transport(), orb(server.supervisor()) { init_rpc(); orb.Listen(0); spec = vespalib::make_string("tcp/localhost:%d", orb.GetListenPort()); - transport.Start(&threadPool); + transport.Start(); } void RPC_waitFor(FRT_RPCRequest *req) { diff --git a/config/src/tests/frt/frt.cpp b/config/src/tests/frt/frt.cpp index 4e77b289f49..c5098d0e7a1 100644 --- a/config/src/tests/frt/frt.cpp +++ b/config/src/tests/frt/frt.cpp @@ -96,7 +96,7 @@ struct Response { ~RPCFixture() { for (size_t i = 0; i < requests.size(); i++) { - requests[i]->SubRef(); + requests[i]->internal_subref(); } } }; @@ -340,7 +340,7 @@ struct V3RequestFixture { } ~V3RequestFixture() { - req->SubRef(); + req->internal_subref(); } void encodePayload(const char * payload, uint32_t payloadSize, uint32_t uncompressedSize, const CompressionType & compressionType) { diff --git a/config/src/tests/frtconnectionpool/frtconnectionpool.cpp b/config/src/tests/frtconnectionpool/frtconnectionpool.cpp index 9c3498a92a1..834b1797ed0 100644 --- a/config/src/tests/frtconnectionpool/frtconnectionpool.cpp +++ b/config/src/tests/frtconnectionpool/frtconnectionpool.cpp @@ -4,7 +4,6 @@ #include <vespa/config/frt/frtconnectionpool.h> #include <vespa/fnet/frt/error.h> #include <vespa/fnet/transport.h> -#include <vespa/fastos/thread.h> #include <sstream> #include <set> #include <unistd.h> @@ -14,7 +13,6 @@ using namespace config; class Test : public vespalib::TestApp { private: static ServerSpec::HostSpecList _sources; - FastOS_ThreadPool _threadPool; FNET_Transport _transport; void verifyAllSourcesInRotation(FRTConnectionPool& sourcePool); public: @@ -33,10 +31,9 @@ public: Test::Test() : vespalib::TestApp(), - _threadPool(), _transport() { - _transport.Start(&_threadPool); + _transport.Start(); } Test::~Test() { diff --git a/config/src/tests/functiontest/functiontest.cpp b/config/src/tests/functiontest/functiontest.cpp index 333645176a0..9d6605be9c1 100644 --- a/config/src/tests/functiontest/functiontest.cpp +++ b/config/src/tests/functiontest/functiontest.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/testkit/test_kit.h> #include <fstream> +#include <cinttypes> #include <vespa/log/log.h> diff --git a/config/src/vespa/config/file_acquirer/file_acquirer.cpp b/config/src/vespa/config/file_acquirer/file_acquirer.cpp index 048e9544dab..f38875b5727 100644 --- a/config/src/vespa/config/file_acquirer/file_acquirer.cpp +++ b/config/src/vespa/config/file_acquirer/file_acquirer.cpp @@ -5,7 +5,6 @@ #include <vespa/fnet/frt/target.h> #include <vespa/fnet/frt/rpcrequest.h> #include <vespa/fnet/transport.h> -#include <vespa/fastos/thread.h> #include <vespa/log/log.h> LOG_SETUP(".config.file_acquirer"); @@ -32,8 +31,8 @@ RpcFileAcquirer::wait_for(const vespalib::string &file_ref, double timeout_s) LOG(warning, "could not acquire file '%s' (%d: %s)", file_ref.c_str(), req->GetErrorCode(), req->GetErrorMessage()); } - req->SubRef(); - target->SubRef(); + req->internal_subref(); + target->internal_subref(); return path; } diff --git a/config/src/vespa/config/frt/frtconfigagent.cpp b/config/src/vespa/config/frt/frtconfigagent.cpp index b5bcb3591f0..15b605ad690 100644 --- a/config/src/vespa/config/frt/frtconfigagent.cpp +++ b/config/src/vespa/config/frt/frtconfigagent.cpp @@ -4,6 +4,7 @@ #include <vespa/config/common/trace.h> #include <vespa/config/common/configresponse.h> #include <vespa/config/common/iconfigholder.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".config.frt.frtconfigagent"); diff --git a/config/src/vespa/config/frt/frtconfigrequest.cpp b/config/src/vespa/config/frt/frtconfigrequest.cpp index 2106da64857..e7da9f53e97 100644 --- a/config/src/vespa/config/frt/frtconfigrequest.cpp +++ b/config/src/vespa/config/frt/frtconfigrequest.cpp @@ -17,7 +17,7 @@ FRTConfigRequest::FRTConfigRequest(Connection * connection, const ConfigKey & ke FRTConfigRequest::~FRTConfigRequest() { - _request->SubRef(); + _request->internal_subref(); } bool diff --git a/config/src/vespa/config/frt/frtconfigresponse.cpp b/config/src/vespa/config/frt/frtconfigresponse.cpp index 43ae3f9b192..28c2825f71c 100644 --- a/config/src/vespa/config/frt/frtconfigresponse.cpp +++ b/config/src/vespa/config/frt/frtconfigresponse.cpp @@ -10,12 +10,12 @@ FRTConfigResponse::FRTConfigResponse(FRT_RPCRequest * request) _responseState(EMPTY), _returnValues(_request->GetReturn()) { - _request->AddRef(); + _request->internal_addref(); } FRTConfigResponse::~FRTConfigResponse() { - _request->SubRef(); + _request->internal_subref(); } bool diff --git a/config/src/vespa/config/frt/frtconnection.cpp b/config/src/vespa/config/frt/frtconnection.cpp index be8ccead65c..ff2a82f855b 100644 --- a/config/src/vespa/config/frt/frtconnection.cpp +++ b/config/src/vespa/config/frt/frtconnection.cpp @@ -30,7 +30,7 @@ FRTConnection::~FRTConnection() { if (_target != nullptr) { LOG(debug, "Shutting down %s", _address.c_str()); - _target->SubRef(); + _target->internal_subref(); _target = nullptr; } } @@ -42,10 +42,10 @@ FRTConnection::getTarget() if (_target == nullptr) { _target = _supervisor.GetTarget(_address.c_str()); } else if ( ! _target->IsValid()) { - _target->SubRef(); + _target->internal_subref(); _target = _supervisor.GetTarget(_address.c_str()); } - _target->AddRef(); + _target->internal_addref(); return _target; } @@ -54,7 +54,7 @@ FRTConnection::invoke(FRT_RPCRequest * req, duration timeout, FRT_IRequestWait * { FRT_Target * target = getTarget(); target->InvokeAsync(req, vespalib::to_s(timeout), waiter); - target->SubRef(); + target->internal_subref(); } void diff --git a/config/src/vespa/config/frt/frtconnectionpool.cpp b/config/src/vespa/config/frt/frtconnectionpool.cpp index 21d6f0dbe90..b73a28483e6 100644 --- a/config/src/vespa/config/frt/frtconnectionpool.cpp +++ b/config/src/vespa/config/frt/frtconnectionpool.cpp @@ -5,7 +5,6 @@ #include <vespa/vespalib/util/size_literals.h> #include <vespa/fnet/frt/supervisor.h> #include <vespa/fnet/transport.h> -#include <vespa/fastos/thread.h> #include <vespa/log/log.h> LOG_SETUP(".config.frt.frtconnectionpool"); @@ -167,14 +166,12 @@ FRTConnectionPool::getScheduler() { return _supervisor->GetScheduler(); } -FRTConnectionPoolWithTransport::FRTConnectionPoolWithTransport(std::unique_ptr<FastOS_ThreadPool> threadPool, - std::unique_ptr<FNET_Transport> transport, +FRTConnectionPoolWithTransport::FRTConnectionPoolWithTransport(std::unique_ptr<FNET_Transport> transport, const ServerSpec & spec, const TimingValues & timingValues) - : _threadPool(std::move(threadPool)), - _transport(std::move(transport)), - _connectionPool(std::make_unique<FRTConnectionPool>(*_transport, spec, timingValues)) + : _transport(std::move(transport)), + _connectionPool(std::make_unique<FRTConnectionPool>(*_transport, spec, timingValues)) { - _transport->Start(_threadPool.get()); + _transport->Start(); } FRTConnectionPoolWithTransport::~FRTConnectionPoolWithTransport() diff --git a/config/src/vespa/config/frt/frtconnectionpool.h b/config/src/vespa/config/frt/frtconnectionpool.h index 564c6506159..5d97f2ae338 100644 --- a/config/src/vespa/config/frt/frtconnectionpool.h +++ b/config/src/vespa/config/frt/frtconnectionpool.h @@ -8,7 +8,6 @@ #include <map> class FNET_Transport; -class FastOS_ThreadPool; namespace config { @@ -103,8 +102,7 @@ public: class FRTConnectionPoolWithTransport : public ConnectionFactory { public: - FRTConnectionPoolWithTransport(std::unique_ptr<FastOS_ThreadPool> threadPool, - std::unique_ptr<FNET_Transport> transport, + FRTConnectionPoolWithTransport(std::unique_ptr<FNET_Transport> transport, const ServerSpec & spec, const TimingValues & timingValues); FRTConnectionPoolWithTransport(const FRTConnectionPoolWithTransport&) = delete; FRTConnectionPoolWithTransport& operator=(const FRTConnectionPoolWithTransport&) = delete; @@ -113,7 +111,6 @@ public: void syncTransport() override { _connectionPool->syncTransport(); } Connection* getCurrent() override { return _connectionPool->getCurrent(); } private: - std::unique_ptr<FastOS_ThreadPool> _threadPool; std::unique_ptr<FNET_Transport> _transport; std::unique_ptr<FRTConnectionPool> _connectionPool; }; diff --git a/config/src/vespa/config/set/configsetsource.cpp b/config/src/vespa/config/set/configsetsource.cpp index 41886a12d01..b84f6411855 100644 --- a/config/src/vespa/config/set/configsetsource.cpp +++ b/config/src/vespa/config/set/configsetsource.cpp @@ -4,6 +4,7 @@ #include <vespa/config/print/asciiconfigwriter.h> #include <vespa/config/common/iconfigholder.h> #include <vespa/config/common/exceptions.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".config.set.configsetsource"); diff --git a/config/src/vespa/config/subscription/configsubscriptionset.cpp b/config/src/vespa/config/subscription/configsubscriptionset.cpp index 4f7bf64f603..9c09a508fff 100644 --- a/config/src/vespa/config/subscription/configsubscriptionset.cpp +++ b/config/src/vespa/config/subscription/configsubscriptionset.cpp @@ -6,6 +6,7 @@ #include <vespa/config/common/misc.h> #include <vespa/config/common/iconfigmanager.h> #include <vespa/config/common/iconfigcontext.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".config.subscription.configsubscriptionset"); diff --git a/config/src/vespa/config/subscription/sourcespec.cpp b/config/src/vespa/config/subscription/sourcespec.cpp index c05f639f9ba..0ab0806885f 100644 --- a/config/src/vespa/config/subscription/sourcespec.cpp +++ b/config/src/vespa/config/subscription/sourcespec.cpp @@ -13,7 +13,6 @@ #include <vespa/config/print/asciiconfigwriter.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/fnet/transport.h> -#include <vespa/fastos/thread.h> #include <cassert> namespace config { @@ -127,8 +126,7 @@ ServerSpec::createSourceFactory(const TimingValues & timingValues) const { const auto vespaVersion = VespaVersion::getCurrentVersion(); return std::make_unique<FRTSourceFactory>( - std::make_unique<FRTConnectionPoolWithTransport>(std::make_unique<FastOS_ThreadPool>(), - std::make_unique<FNET_Transport>(), + std::make_unique<FRTConnectionPoolWithTransport>(std::make_unique<FNET_Transport>(), *this, timingValues), timingValues, _traceLevel, vespaVersion, _compressionType); } diff --git a/configd/src/apps/cmd/main.cpp b/configd/src/apps/cmd/main.cpp index 3ea26fee150..9ee7130a06e 100644 --- a/configd/src/apps/cmd/main.cpp +++ b/configd/src/apps/cmd/main.cpp @@ -74,7 +74,7 @@ void Cmd::finiRPC() { if (_target != nullptr) { - _target->SubRef(); + _target->internal_subref(); _target = nullptr; } _server.reset(); @@ -150,7 +150,7 @@ Cmd::run(const Method &cmd, const char *arg) } } } - req->SubRef(); + req->internal_subref(); finiRPC(); return retval; } diff --git a/configd/src/apps/sentinel/config-owner.cpp b/configd/src/apps/sentinel/config-owner.cpp index 90ceb705dc7..1fe361ef739 100644 --- a/configd/src/apps/sentinel/config-owner.cpp +++ b/configd/src/apps/sentinel/config-owner.cpp @@ -2,6 +2,7 @@ #include "config-owner.h" #include <vespa/config/subscription/configsubscriber.hpp> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".sentinel.config-owner"); @@ -9,7 +10,6 @@ LOG_SETUP(".sentinel.config-owner"); namespace config::sentinel { ConfigOwner::ConfigOwner() = default; - ConfigOwner::~ConfigOwner() = default; void diff --git a/configd/src/apps/sentinel/connectivity.cpp b/configd/src/apps/sentinel/connectivity.cpp index b9641cc19e0..9cd8d5c985a 100644 --- a/configd/src/apps/sentinel/connectivity.cpp +++ b/configd/src/apps/sentinel/connectivity.cpp @@ -75,10 +75,9 @@ void classifyConnFails(ConnectivityMap &connectivityMap, LOG_ASSERT(cmIter != connectivityMap.end()); OutwardCheckContext cornerContext(goodNeighborSpecs.size(), nameToCheck, portToCheck, rpcServer.orb()); ConnectivityMap cornerProbes; - int ping_timeout = 1000; + int ping_timeout = 1000 + 50 * goodNeighborSpecs.size(); for (const auto & hp : goodNeighborSpecs) { cornerProbes.try_emplace(hp.first, spec(hp), cornerContext, ping_timeout); - ping_timeout += 20; } cornerContext.latch.await(); size_t numReportsUp = 0; @@ -154,10 +153,9 @@ Connectivity::checkConnectivity(RpcServer &rpcServer) { rpcServer.getPort(), rpcServer.orb()); ConnectivityMap connectivityMap; - int ping_timeout = 1000; + int ping_timeout = 1000 + 50 * _checkSpecs.size(); for (const auto &host_and_port : _checkSpecs) { connectivityMap.try_emplace(host_and_port.first, spec(host_and_port), checkContext, ping_timeout); - ping_timeout += 20; } checkContext.latch.await(); classifyConnFails(connectivityMap, _checkSpecs, rpcServer); diff --git a/configd/src/apps/sentinel/logctl.cpp b/configd/src/apps/sentinel/logctl.cpp index 94759d4f102..057178cec2e 100644 --- a/configd/src/apps/sentinel/logctl.cpp +++ b/configd/src/apps/sentinel/logctl.cpp @@ -5,7 +5,7 @@ #include <sys/types.h> #include <sys/wait.h> #include <unistd.h> -#include <stdio.h> +#include <cstring> #include <vespa/log/log.h> LOG_SETUP(".sentinel.logctl"); diff --git a/configd/src/apps/sentinel/model-owner.cpp b/configd/src/apps/sentinel/model-owner.cpp index 5dd7a033933..93024911f4e 100644 --- a/configd/src/apps/sentinel/model-owner.cpp +++ b/configd/src/apps/sentinel/model-owner.cpp @@ -3,6 +3,7 @@ #include "model-owner.h" #include <vespa/config/common/exceptions.h> #include <vespa/config/subscription/configsubscriber.hpp> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".sentinel.model-owner"); diff --git a/configd/src/apps/sentinel/outward-check.cpp b/configd/src/apps/sentinel/outward-check.cpp index b81c8e23750..b8bb2e69077 100644 --- a/configd/src/apps/sentinel/outward-check.cpp +++ b/configd/src/apps/sentinel/outward-check.cpp @@ -53,9 +53,9 @@ void OutwardCheck::RequestDone(FRT_RPCRequest *req) { req->GetErrorMessage(), req->GetErrorCode()); _result = CcResult::CONN_FAIL; } - _req->SubRef(); + _req->internal_subref(); _req = nullptr; - _target->SubRef(); + _target->internal_subref(); _target = nullptr; _context.latch.countDown(); } diff --git a/configd/src/apps/sentinel/peer-check.cpp b/configd/src/apps/sentinel/peer-check.cpp index 0af5a6fb58f..ac3775d4c4d 100644 --- a/configd/src/apps/sentinel/peer-check.cpp +++ b/configd/src/apps/sentinel/peer-check.cpp @@ -39,9 +39,9 @@ void PeerCheck::RequestDone(FRT_RPCRequest *req) { LOG(debug, "OK ping to %s [port %d]", _hostname.c_str(), _portnum); _statusOk = true; } - _req->SubRef(); + _req->internal_subref(); _req = nullptr; - _target->SubRef(); + _target->internal_subref(); _target = nullptr; // Note: will delete this object, so must be called as final step: _callback.returnStatus(_statusOk); diff --git a/configd/src/apps/sentinel/report-connectivity.cpp b/configd/src/apps/sentinel/report-connectivity.cpp index 8417384dda9..d059980aae9 100644 --- a/configd/src/apps/sentinel/report-connectivity.cpp +++ b/configd/src/apps/sentinel/report-connectivity.cpp @@ -22,9 +22,9 @@ ReportConnectivity::ReportConnectivity(FRT_RPCRequest *req, int timeout_ms, FRT_ auto map = Connectivity::specsFrom(cfg.value()); LOG(debug, "making connectivity report for %zd peers", map.size()); _remaining = map.size(); + timeout_ms += 50 * map.size(); for (const auto & [ hostname, port ] : map) { _checks.emplace_back(std::make_unique<PeerCheck>(*this, hostname, port, orb, timeout_ms)); - timeout_ms += 20; } } else { _parentRequest->SetError(FRTE_RPC_METHOD_FAILED, "failed getting model config"); diff --git a/configdefinitions/CMakeLists.txt b/configdefinitions/CMakeLists.txt index c374e93904e..80b53d1dc0a 100644 --- a/configdefinitions/CMakeLists.txt +++ b/configdefinitions/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalib config_cloudconfig diff --git a/configdefinitions/src/vespa/athenz-provider-service.def b/configdefinitions/src/vespa/athenz-provider-service.def index 2131aa88d30..4c9c74f9b8f 100644 --- a/configdefinitions/src/vespa/athenz-provider-service.def +++ b/configdefinitions/src/vespa/athenz-provider-service.def @@ -13,6 +13,11 @@ secretName string # Secret version secretVersion int +# Tempory resources +sisSecretName string default="" +sisSecretVersion int default=0 +sisUrl string default = "" + # Secret name of CA certificate caCertSecretName string diff --git a/configserver/CMakeLists.txt b/configserver/CMakeLists.txt index f189dc4f2c1..201d419c669 100644 --- a/configserver/CMakeLists.txt +++ b/configserver/CMakeLists.txt @@ -12,7 +12,6 @@ install(DIRECTORY DESTINATION conf/configserver) install(DIRECTORY DESTINATION conf/configserver-app/components) install(DIRECTORY DESTINATION conf/configserver-app/config-models) -install_symlink(lib/jars/athenz-identity-provider-service-jar-with-dependencies.jar conf/configserver-app/components/athenz-identity-provider-service.jar) install_symlink(lib/jars/config-model-fat.jar conf/configserver-app/components/config-model-fat.jar) install_symlink(lib/jars/configserver-flags-jar-with-dependencies.jar conf/configserver-app/components/configserver-flags.jar) install_symlink(lib/jars/flags-jar-with-dependencies.jar conf/configserver-app/components/flags.jar) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 9865cda7bc9..8a4d523a6e4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -4,8 +4,8 @@ package com.yahoo.vespa.config.server; import ai.vespa.http.DomainName; import ai.vespa.http.HttpURL; import ai.vespa.http.HttpURL.Query; -import ai.vespa.util.http.hc5.VespaHttpClientBuilder; import ai.vespa.util.http.hc5.DefaultHttpClientBuilder; +import ai.vespa.util.http.hc5.VespaHttpClientBuilder; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.component.annotation.Inject; @@ -94,10 +94,8 @@ import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.Orchestrator; import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; import org.apache.hc.core5.http.HttpHeaders; import org.apache.hc.core5.http.message.BasicHeader; - import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -229,7 +227,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // Should be used by tests only (first constructor in this class makes sure we use injectable components where possible) public static class Builder { private TenantRepository tenantRepository; - private Optional<Provisioner> hostProvisioner; private HttpProxy httpProxy = new HttpProxy(new SimpleHttpFetcher(Duration.ofSeconds(30))); private EndpointsChecker endpointsChecker = __ -> { throw new UnsupportedOperationException(); }; private Clock clock = Clock.systemUTC(); @@ -252,18 +249,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return this; } - public Builder withProvisioner(Provisioner provisioner) { - if (this.hostProvisioner != null) throw new IllegalArgumentException("provisioner already set in builder"); - this.hostProvisioner = Optional.ofNullable(provisioner); - return this; - } - - public Builder withHostProvisionerProvider(HostProvisionerProvider hostProvisionerProvider) { - if (this.hostProvisioner != null) throw new IllegalArgumentException("provisioner already set in builder"); - this.hostProvisioner = hostProvisionerProvider.getHostProvisioner(); - return this; - } - public Builder withHttpProxy(HttpProxy httpProxy) { this.httpProxy = httpProxy; return this; @@ -316,7 +301,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public ApplicationRepository build() { return new ApplicationRepository(tenantRepository, - hostProvisioner, + tenantRepository.hostProvisionerProvider().getHostProvisioner(), InfraDeployerProvider.empty().getInfraDeployer(), configConvergenceChecker, httpProxy, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigActivationListener.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigActivationListener.java index f7e9e270b9c..e52089f5400 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigActivationListener.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigActivationListener.java @@ -4,35 +4,17 @@ package com.yahoo.vespa.config.server; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.server.application.ApplicationSet; -import java.util.Collection; - /** * A ConfigActivationListener is used to signal to a component that config has been - * activated. It only exists because the RpcServer cannot distinguish between a - * successful activation of a new application and an activation of the same application. + * activated for an application or that an application has been removed. It only exists + * because the RpcServer cannot distinguish between a successful activation of a new + * application and an activation of the same application. * * @author Ulf Lilleengen */ public interface ConfigActivationListener { /** - * Signals the listener that hosts used by a particular tenant. - * - * @param applicationId application id - * @param newHosts a {@link Collection} of hosts used by tenant. - */ - void hostsUpdated(ApplicationId applicationId, Collection<String> newHosts); - - /** - * Verifies that given hosts are available for use by tenant. - * - * @param applicationId application id - * @param newHosts a {@link java.util.Collection} of hosts that tenant wants to allocate. - * @throws java.lang.IllegalArgumentException if one or more of the hosts are in use by another tenant. - */ - void verifyHostsAreAvailable(ApplicationId applicationId, Collection<String> newHosts); - - /** * Configs has been activated for an application: Either an application * has been deployed for the first time, or it has been externally or internally redeployed. * diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index b811fe21310..885aedb0cbe 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.AbstractComponent; +import com.yahoo.component.Version; import com.yahoo.component.annotation.Inject; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.provision.ApplicationId; @@ -127,12 +128,13 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable public void start() { startRpcServerWithFileDistribution(); // No config requests allowed yet, will be allowed after bootstrapping done if (versionState.isUpgraded()) { - log.log(Level.INFO, "Config server upgrading from " + versionState.storedVersion() + " to " - + versionState.currentVersion() + ". Redeploying all applications"); + if (versionState.storedVersion() != Version.emptyVersion) + log.log(Level.INFO, "Config server upgrading from " + versionState.storedVersion() + " to " + + versionState.currentVersion() + ". Redeploying all applications"); try { redeployAllApplications(); versionState.saveNewVersion(); - log.log(Level.INFO, "All applications redeployed successfully"); + log.log(Level.FINE, "All applications redeployed successfully"); } catch (Exception e) { log.log(Level.SEVERE, "Redeployment of applications failed", e); redeployingApplicationsFailed(); @@ -217,14 +219,16 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable new DaemonThreadFactory("redeploy-apps-")); // Keep track of deployment status per application Map<ApplicationId, Future<?>> deployments = new HashMap<>(); - log.log(Level.INFO, () -> "Redeploying " + applicationIds.size() + " apps " + applicationIds + " with " + - configserverConfig.numRedeploymentThreads() + " threads"); - applicationIds.forEach(appId -> deployments.put(appId, executor.submit(() -> { - log.log(Level.INFO, () -> "Starting redeployment of " + appId); - applicationRepository.deployFromLocalActive(appId, true /* bootstrap */) - .ifPresent(Deployment::activate); - log.log(Level.INFO, () -> appId + " redeployed"); - }))); + if (applicationIds.size() > 0) { + log.log(Level.INFO, () -> "Redeploying " + applicationIds.size() + " apps " + applicationIds + " with " + + configserverConfig.numRedeploymentThreads() + " threads"); + applicationIds.forEach(appId -> deployments.put(appId, executor.submit(() -> { + log.log(Level.INFO, () -> "Starting redeployment of " + appId); + applicationRepository.deployFromLocalActive(appId, true /* bootstrap */) + .ifPresent(Deployment::activate); + log.log(Level.INFO, () -> appId + " redeployed"); + }))); + } List<ApplicationId> failedDeployments = checkDeployments(deployments); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java b/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java index a614e251dca..89f7755729c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java @@ -38,9 +38,15 @@ public class GetConfigContext { return new GetConfigContext(app, handler, trace); } + public static GetConfigContext empty() { + return new GetConfigContext(null, null, null); + } + public static GetConfigContext testContext(ApplicationId app) { return new GetConfigContext(app, null, null); } + + public boolean isEmpty() { return app == null && requestHandler == null && trace == null; } /** * Helper to produce a log preamble with the tenant and app id diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationSet.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationSet.java index ad50117e327..5650c2e7e15 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationSet.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationSet.java @@ -1,16 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; +import com.yahoo.component.Version; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.component.Version; - import java.time.Instant; -import java.util.*; -import java.util.stream.Collectors; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; /** - * Immutable set of {@link Application}s with the same {@link ApplicationId}. With methods for getting defaults. + * Immutable set of {@link Application}s with the same {@link ApplicationId}, applications have difference vespa versions. * * @author vegard */ @@ -42,8 +44,17 @@ public final class ApplicationSet { latestVersion = this.applications.keySet().stream().max(Version::compareTo).get(); } + public static ApplicationSet fromList(List<Application> applications) { + return new ApplicationSet(applications); + } + + // For testing + public static ApplicationSet from(Application application) { + return fromList(List.of(application)); + } + /** - * Returns the specified version, or the latest if not specified, or if the given version is not + * Returns an application for the specified version, or the latest if not specified, or if the given version is not * available and the latest is a permissible substitution. * * @throws VersionDoesNotExistException if the specified version is not available and the latest version is not @@ -68,21 +79,13 @@ public final class ApplicationSet { return Optional.empty(); } - /** Returns the application for the given version, if available */ + /** Returns the application for the given version, or empty if not found */ public Optional<Application> get(Version version) { return Optional.ofNullable(applications.get(version)); } public ApplicationId getId() { return applicationId; } - public static ApplicationSet fromList(List<Application> applications) { - return new ApplicationSet(applications); - } - - public static ApplicationSet from(Application application) { - return fromList(List.of(application)); - } - public Collection<String> getAllHosts() { return applications.values().stream() .flatMap(app -> app.getModel().getHosts().stream() @@ -91,9 +94,7 @@ public final class ApplicationSet { } public void updateHostMetrics() { - for (Application application : applications.values()) { - application.updateHostMetrics(application.getModel().getHosts().size()); - } + applications.values().forEach(app -> app.updateHostMetrics(app.getModel().getHosts().size())); } public long getApplicationGeneration() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 5831cb3e75f..09a687657c6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -216,10 +216,10 @@ public class TenantApplications implements RequestHandler, HostValidator { } private void notifyConfigActivationListeners(ApplicationSet applicationSet) { - if (applicationSet.getAllApplications().isEmpty()) throw new IllegalArgumentException("application set cannot be empty"); + List<Application> applications = applicationSet.getAllApplications(); + if (applications.isEmpty()) throw new IllegalArgumentException("application set cannot be empty"); - configActivationListener.hostsUpdated(applicationSet.getAllApplications().get(0).toApplicationInfo().getApplicationId(), - applicationSet.getAllHosts()); + hostRegistry.update(applications.get(0).getId(), applicationSet.getAllHosts()); configActivationListener.configActivated(applicationSet); } @@ -253,7 +253,7 @@ public class TenantApplications implements RequestHandler, HostValidator { if (hasApplication(applicationId)) { applicationMapper.remove(applicationId); - hostRegistry.removeHostsForKey(applicationId); + hostRegistry.removeHosts(applicationId); configActivationListenersOnRemove(applicationId); tenantMetricUpdater.setApplications(applicationMapper.numApplications()); metrics.removeMetricUpdater(Metrics.createDimensions(applicationId)); @@ -277,17 +277,17 @@ public class TenantApplications implements RequestHandler, HostValidator { } private void configActivationListenersOnRemove(ApplicationId applicationId) { - configActivationListener.hostsUpdated(applicationId, hostRegistry.getHostsForKey(applicationId)); + hostRegistry.removeHosts(applicationId); configActivationListener.applicationRemoved(applicationId); } private void setActiveApp(ApplicationSet applicationSet) { - ApplicationId id = applicationSet.getId(); + ApplicationId applicationId = applicationSet.getId(); Collection<String> hostsForApp = applicationSet.getAllHosts(); - hostRegistry.update(id, hostsForApp); + hostRegistry.update(applicationId, hostsForApp); applicationSet.updateHostMetrics(); tenantMetricUpdater.setApplications(applicationMapper.numApplications()); - applicationMapper.register(id, applicationSet); + applicationMapper.register(applicationId, applicationSet); } @Override @@ -377,7 +377,7 @@ public class TenantApplications implements RequestHandler, HostValidator { @Override public ApplicationId resolveApplicationId(String hostName) { - return hostRegistry.getKeyForHost(hostName); + return hostRegistry.getApplicationId(hostName); } @Override @@ -399,11 +399,11 @@ public class TenantApplications implements RequestHandler, HostValidator { @Override public void verifyHosts(ApplicationId applicationId, Collection<String> newHosts) { hostRegistry.verifyHosts(applicationId, newHosts); - configActivationListener.verifyHostsAreAvailable(applicationId, newHosts); } + // TODO: Duplicate of resolveApplicationId() above public ApplicationId getApplicationIdForHostName(String hostname) { - return hostRegistry.getKeyForHost(hostname); + return hostRegistry.getApplicationId(hostname); } public TenantFileSystemDirs getTenantFileSystemDirs() { return tenantFileSystemDirs; } @@ -418,8 +418,9 @@ public class TenantApplications implements RequestHandler, HostValidator { /** * Waiter for removing application. Will wait for some time for all servers to remove application, - * but will accept majority of servers to have removed app if it takes a long time. + * but will accept the majority of servers to have removed app if it takes a long time. */ + // TODO: Merge with CuratorCompletionWaiter static class RemoveApplicationWaiter implements CompletionWaiter { private static final java.util.logging.Logger log = Logger.getLogger(RemoveApplicationWaiter.class.getName()); @@ -485,7 +486,7 @@ public class TenantApplications implements RequestHandler, HostValidator { gotQuorumTime = clock.instant(); // Give up if more than some time has passed since we got quorum, otherwise continue - if (Duration.between(Instant.now(), gotQuorumTime.plus(waitForAll)).isNegative()) { + if (Duration.between(clock.instant(), gotQuorumTime.plus(waitForAll)).isNegative()) { logBarrierCompleted(respondents, startTime); break; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index df449ca017b..66a5bc5a023 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -1,13 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.deploy; -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; @@ -29,11 +28,14 @@ import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.config.server.session.SessionRepository; import com.yahoo.vespa.config.server.tenant.Tenant; +import com.yahoo.yolean.concurrent.Memoized; + import java.time.Clock; import java.time.Duration; import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -113,8 +115,6 @@ public class Deployment implements com.yahoo.config.provision.Deployment { deleteSession(); throw e; } - - waitForResourcesOrTimeout(params, session, provisioner); } /** Activates this. If it is not already prepared, this will call prepare first. */ @@ -125,6 +125,8 @@ public class Deployment implements com.yahoo.config.provision.Deployment { validateSessionStatus(session); PrepareParams params = this.params.get(); + waitForResourcesOrTimeout(params, session, provisioner); + ApplicationId applicationId = session.getApplicationId(); try (ActionTimer timer = applicationRepository.timerFor(applicationId, "deployment.activateMillis")) { TimeoutBudget timeoutBudget = params.getTimeoutBudget(); @@ -263,7 +265,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { // Use supplier because we shouldn't/can't create this before validateSessionStatus() for prepared deployments, // memoize because we want to create this once for unprepared deployments - return Suppliers.memoize(() -> { + return new Memoized<>(() -> { TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); PrepareParams.Builder params = new PrepareParams.Builder() @@ -288,20 +290,19 @@ public class Deployment implements com.yahoo.config.provision.Deployment { Set<HostSpec> preparedHosts = session.getAllocatedHosts().getHosts(); ActivationContext context = new ActivationContext(session.getSessionId()); - ProvisionLock lock = new ProvisionLock(session.getApplicationId(), () -> {}); - AtomicReference<TransientException> lastException = new AtomicReference<>(); + AtomicReference<Exception> lastException = new AtomicReference<>(); while (true) { params.getTimeoutBudget().assertNotTimedOut( () -> "Timeout exceeded while waiting for application resources of '" + session.getApplicationId() + "'" + Optional.ofNullable(lastException.get()).map(e -> ". Last exception: " + e.getMessage()).orElse("")); - try { + try (ProvisionLock lock = provisioner.get().lock(session.getApplicationId())) { // Call to activate to make sure that everything is ready, but do not commit the transaction ApplicationTransaction transaction = new ApplicationTransaction(lock, new NestedTransaction()); provisioner.get().activate(preparedHosts, context, transaction); return; - } catch (TransientException e) { + } catch (ApplicationLockException | TransientException e) { lastException.set(e); try { Thread.sleep(durationBetweenResourceReadyChecks.toMillis()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 1d80740ffe0..ad5423f0a94 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -202,6 +202,7 @@ public class ModelContextImpl implements ModelContext { private final int rpc_events_before_wakeup; private final boolean useRestrictedDataPlaneBindings; private final int heapPercentage; + private final boolean enableGlobalPhase; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -246,6 +247,7 @@ public class ModelContextImpl implements ModelContext { this.queryDispatchWarmup = flagValue(source, appId, version, PermanentFlags.QUERY_DISPATCH_WARMUP); this.useRestrictedDataPlaneBindings = flagValue(source, appId, version, Flags.RESTRICT_DATA_PLANE_BINDINGS); this.heapPercentage = flagValue(source, appId, version, PermanentFlags.HEAP_SIZE_PERCENTAGE); + this.enableGlobalPhase = flagValue(source, appId, version, Flags.ENABLE_GLOBAL_PHASE); } @Override public int heapSizePercentage() { return heapPercentage; } @@ -298,6 +300,7 @@ public class ModelContextImpl implements ModelContext { return defVal; } @Override public boolean useRestrictedDataPlaneBindings() { return useRestrictedDataPlaneBindings; } + @Override public boolean enableGlobalPhase() { return enableGlobalPhase; } private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index 85b30f4d303..b9118602058 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -23,6 +23,7 @@ import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; +import java.time.Clock; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.logging.Level; @@ -153,6 +154,8 @@ public class FileDirectory extends AbstractComponent { return true; } + // update last modified time so that maintainer deleting unused file references considers this as recently used + destinationDir.setLastModified(Clock.systemUTC().instant().toEpochMilli()); log.log(Level.FINE, "Directory for file reference '" + fileReference.value() + "' already exists and has all content"); return false; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java index b89f3bba835..e6f2452b693 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java @@ -23,19 +23,19 @@ public class HostRegistry implements HostValidator { private static final Logger log = Logger.getLogger(HostRegistry.class.getName()); - private final Map<String, ApplicationId> host2KeyMap = new ConcurrentHashMap<>(); + private final Map<String, ApplicationId> host2ApplicationId = new ConcurrentHashMap<>(); - public ApplicationId getKeyForHost(String hostName) { - return host2KeyMap.get(hostName); + public ApplicationId getApplicationId(String hostName) { + return host2ApplicationId.get(hostName); } public synchronized void update(ApplicationId key, Collection<String> newHosts) { verifyHosts(key, newHosts); - Collection<String> currentHosts = getHostsForKey(key); + Collection<String> currentHosts = getHosts(key); log.log(Level.FINE, () -> "Setting hosts for key '" + key + "', " + "newHosts: " + newHosts + ", " + "currentHosts: " + currentHosts); - Collection<String> removedHosts = getRemovedHosts(newHosts, currentHosts); + Collection<String> removedHosts = findRemovedHosts(newHosts, currentHosts); removeHosts(removedHosts); addHosts(key, newHosts); } @@ -45,49 +45,49 @@ public class HostRegistry implements HostValidator { for (String host : newHosts) { if (hostAlreadyTaken(host, applicationId)) { throw new IllegalArgumentException("'" + applicationId + "' tried to allocate host '" + host + - "', but the host is already taken by '" + host2KeyMap.get(host) + "'"); + "', but the host is already taken by '" + host2ApplicationId.get(host) + "'"); } } } - public synchronized void removeHostsForKey(ApplicationId key) { - host2KeyMap.entrySet().removeIf(entry -> entry.getValue().equals(key)); + public synchronized void removeHosts(ApplicationId key) { + host2ApplicationId.entrySet().removeIf(entry -> entry.getValue().equals(key)); } - public synchronized void removeHostsForKey(TenantName key) { - host2KeyMap.entrySet().removeIf(entry -> entry.getValue().tenant().equals(key)); + public synchronized void removeHosts(TenantName key) { + host2ApplicationId.entrySet().removeIf(entry -> entry.getValue().tenant().equals(key)); } public synchronized Collection<String> getAllHosts() { - return Collections.unmodifiableCollection(new ArrayList<>(host2KeyMap.keySet())); + return Collections.unmodifiableCollection(new ArrayList<>(host2ApplicationId.keySet())); } - public synchronized Collection<String> getHostsForKey(ApplicationId key) { - return host2KeyMap.entrySet().stream() - .filter(entry -> entry.getValue().equals(key)) - .map(Map.Entry::getKey) - .collect(Collectors.toSet()); + public synchronized Collection<String> getHosts(ApplicationId key) { + return host2ApplicationId.entrySet().stream() + .filter(entry -> entry.getValue().equals(key)) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); } private boolean hostAlreadyTaken(String host, ApplicationId key) { - return host2KeyMap.containsKey(host) && !key.equals(host2KeyMap.get(host)); + return host2ApplicationId.containsKey(host) && !key.equals(host2ApplicationId.get(host)); } - private static Collection<String> getRemovedHosts(Collection<String> newHosts, Collection<String> previousHosts) { + private static Collection<String> findRemovedHosts(Collection<String> newHosts, Collection<String> previousHosts) { return Collections2.filter(previousHosts, host -> !newHosts.contains(host)); } private void removeHosts(Collection<String> removedHosts) { for (String host : removedHosts) { log.log(Level.FINE, () -> "Removing " + host); - host2KeyMap.remove(host); + host2ApplicationId.remove(host); } } private void addHosts(ApplicationId key, Collection<String> newHosts) { for (String host : newHosts) { log.log(Level.FINE, () -> "Adding " + host); - host2KeyMap.put(host, key); + host2ApplicationId.put(host, key); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java index 3baa57b2b01..e043afdbf43 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java @@ -30,15 +30,18 @@ import java.util.Set; */ public class HttpConfigRequest implements GetConfigRequest, TenantRequest { - private static final String HTTP_PROPERTY_NOCACHE = "noCache"; + private static final String NOCACHE = "noCache"; + private static final String REQUIRED_GENERATION = "requiredGeneration"; private final ConfigKey<?> key; private final ApplicationId appId; private final boolean noCache; + private final Optional<Long> requiredGeneration; - private HttpConfigRequest(ConfigKey<?> key, ApplicationId appId, boolean noCache) { + private HttpConfigRequest(ConfigKey<?> key, ApplicationId appId, boolean noCache, Optional<Long> requiredGeneration) { this.key = key; this.appId = appId; this.noCache = noCache; + this.requiredGeneration = requiredGeneration; } private static ConfigKey<?> fromRequestV1(HttpRequest req) { @@ -59,7 +62,10 @@ public class HttpConfigRequest implements GetConfigRequest, TenantRequest { } public static HttpConfigRequest createFromRequestV1(HttpRequest req) { - return new HttpConfigRequest(fromRequestV1(req), ApplicationId.defaultId(), req.getBooleanProperty(HTTP_PROPERTY_NOCACHE)); + return new HttpConfigRequest(fromRequestV1(req), + ApplicationId.defaultId(), + req.getBooleanProperty(NOCACHE), + requiredGeneration(req)); } public static HttpConfigRequest createFromRequestV2(HttpRequest req) { @@ -89,7 +95,8 @@ public class HttpConfigRequest implements GetConfigRequest, TenantRequest { cNamespace = nns.second; return new HttpConfigRequest(new ConfigKey<>(cName, cId, cNamespace), new ApplicationId.Builder().applicationName(application).tenant(tenant).build(), - req.getBooleanProperty(HTTP_PROPERTY_NOCACHE)); + req.getBooleanProperty(NOCACHE), + requiredGeneration(req)); } // The URL pattern with full app id given @@ -117,7 +124,10 @@ public class HttpConfigRequest implements GetConfigRequest, TenantRequest { .applicationName(application) .instanceName(instance) .build(); - return new HttpConfigRequest(new ConfigKey<>(cName, cId, cNamespace), appId, req.getBooleanProperty(HTTP_PROPERTY_NOCACHE)); + return new HttpConfigRequest(new ConfigKey<>(cName, cId, cNamespace), + appId, + req.getBooleanProperty(NOCACHE), + requiredGeneration(req)); } /** @@ -144,7 +154,11 @@ public class HttpConfigRequest implements GetConfigRequest, TenantRequest { public static void throwModelNotReady() { throw new NotFoundException("Config not available, verify that an application package has been deployed and activated."); } - + + public static void throwPreconditionFailed(long requiredGeneration) { + throw new PreconditionFailedException("Config for required generation " + requiredGeneration + " could not be found."); + } + /** * If the given config is produced by the model at all * @@ -199,4 +213,11 @@ public class HttpConfigRequest implements GetConfigRequest, TenantRequest { @Override public PayloadChecksums configPayloadChecksums() { return PayloadChecksums.empty(); } + public Optional<Long> requiredGeneration() { return requiredGeneration; } + + static Optional<Long> requiredGeneration(HttpRequest req) { + Optional<String> requiredGeneration = Optional.ofNullable(req.getProperty(REQUIRED_GENERATION)); + return requiredGeneration.map(Long::parseLong); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java index 40ce16145e7..3b5269cdf11 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java @@ -16,6 +16,7 @@ import static com.yahoo.jdisc.Response.Status.CONFLICT; import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED; import static com.yahoo.jdisc.Response.Status.NOT_FOUND; +import static com.yahoo.jdisc.Response.Status.PRECONDITION_FAILED; import static com.yahoo.jdisc.Response.Status.REQUEST_TIMEOUT; /** @@ -51,7 +52,8 @@ public class HttpErrorResponse extends HttpResponse { CERTIFICATE_NOT_READY, LOAD_BALANCER_NOT_READY, CONFIG_NOT_CONVERGED, - REINDEXING_STATUS_UNAVAILABLE + REINDEXING_STATUS_UNAVAILABLE, + PRECONDITION_FAILED } public static HttpErrorResponse notFoundError(String msg) { @@ -114,6 +116,10 @@ public class HttpErrorResponse extends HttpResponse { return new HttpErrorResponse(CONFLICT, ErrorCode.REINDEXING_STATUS_UNAVAILABLE.name(), msg); } + public static HttpResponse preconditionFailed(String msg) { + return new HttpErrorResponse(PRECONDITION_FAILED, ErrorCode.PRECONDITION_FAILED.name(), msg); + } + @Override public void render(OutputStream stream) throws IOException { new JsonFormat(true).encode(stream, slime); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java index dc3a05e65f9..a0e814f32d8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java @@ -71,6 +71,8 @@ public class HttpHandler extends ThreadedHttpRequestHandler { return HttpErrorResponse.loadBalancerNotReady(getMessage(e, request)); } catch (ReindexingStatusException e) { return HttpErrorResponse.reindexingStatusUnavailable(getMessage(e, request)); + } catch (PreconditionFailedException e) { + return HttpErrorResponse.preconditionFailed(getMessage(e, request)); } catch (Exception e) { log.log(Level.WARNING, "Unexpected exception handling a config server request", e); return HttpErrorResponse.internalServerError(getMessage(e, request)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/NotFoundException.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/NotFoundException.java index eb008de6ee5..688890c75b2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/NotFoundException.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/NotFoundException.java @@ -5,7 +5,6 @@ package com.yahoo.vespa.config.server.http; * Exception that will create a http response with NOT_FOUND response code (404) * * @author hmusum - * @since 5.1.17 */ public class NotFoundException extends RuntimeException { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/PreconditionFailedException.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/PreconditionFailedException.java new file mode 100644 index 00000000000..ef3924425c2 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/PreconditionFailedException.java @@ -0,0 +1,16 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +/** + * Exception that will create a http response with NOT_FOUND response code (404) + * + * @author hmusum + */ +public class PreconditionFailedException extends RuntimeException { + + public PreconditionFailedException(String message) { + super(message); + } + +} + diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java index ecea7422ce8..98a65fca987 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java @@ -100,6 +100,10 @@ public class ApplicationApiHandler extends SessionHandler { compressedStream = createFromCompressedStream(request.getData(), request.getHeader(contentTypeHeader), maxApplicationPackageSize); } + // Aid debugging by adding full application id to access log (since only tenant name is part of the request URI path) + request.getAccessLogEntry() + .ifPresent(e -> e.addKeyValue("app.id", prepareParams.getApplicationId().toFullString())); + try (compressedStream) { PrepareResult result = applicationRepository.deploy(compressedStream, prepareParams); return new SessionPrepareAndActivateResponse(result, request, prepareParams.getApplicationId(), zone); @@ -111,7 +115,7 @@ public class ApplicationApiHandler extends SessionHandler { @Override public Duration getTimeout() { - return zookeeperBarrierTimeout.plus(Duration.ofSeconds(30)); + return zookeeperBarrierTimeout.plus(Duration.ofSeconds(120)); } private TenantName validateTenant(HttpRequest request) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java index 3ab3df99a10..0389b2a6c98 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java @@ -4,31 +4,27 @@ package com.yahoo.vespa.config.server.http.v2; import com.yahoo.component.annotation.Inject; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import java.util.logging.Level; import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.server.RequestHandler; -import com.yahoo.vespa.config.server.http.v2.request.HttpConfigRequests; -import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.http.HttpConfigRequest; import com.yahoo.vespa.config.server.http.HttpConfigResponse; import com.yahoo.vespa.config.server.http.HttpHandler; - +import com.yahoo.vespa.config.server.http.v2.request.HttpConfigRequests; +import com.yahoo.vespa.config.server.tenant.TenantRepository; import java.util.Optional; +import java.util.logging.Level; /** * HTTP handler for a getConfig operation * * @author Ulf Lilleengen - * @since 5.1 */ public class HttpGetConfigHandler extends HttpHandler { private final TenantRepository tenantRepository; @Inject - public HttpGetConfigHandler(HttpHandler.Context ctx, - TenantRepository tenantRepository) - { + public HttpGetConfigHandler(HttpHandler.Context ctx, TenantRepository tenantRepository) { super(ctx); this.tenantRepository = tenantRepository; } @@ -45,6 +41,8 @@ public class HttpGetConfigHandler extends HttpHandler { log.log(Level.FINE, () -> "nocache=" + request.noCache()); ConfigResponse config = requestHandler.resolveConfig(request.getApplicationId(), request, Optional.empty()); if (config == null) HttpConfigRequest.throwModelNotReady(); + if (request.requiredGeneration().isPresent() && request.requiredGeneration().get() != config.getGeneration()) + HttpConfigRequest.throwPreconditionFailed(request.requiredGeneration().get()); return config; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java index 9229fb88b40..5547156721a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java @@ -42,11 +42,6 @@ public class HostProvisionerProvider { } // for testing - public static HostProvisionerProvider withProvisioner(Provisioner provisioner, boolean hostedVespa) { - return withProvisioner(provisioner, new ConfigserverConfig(new ConfigserverConfig.Builder().hostedVespa(hostedVespa))); - } - - // for testing public static HostProvisionerProvider withProvisioner(Provisioner provisioner, ConfigserverConfig config) { ComponentRegistry<Provisioner> registry = new ComponentRegistry<>(); registry.register(ComponentId.createAnonymousComponentId("foobar"), provisioner); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java index 1c419ce047a..c916a429599 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.config.server.rpc; import com.yahoo.cloud.config.SentinelConfig; -import com.yahoo.collections.Pair; import com.yahoo.component.Version; import com.yahoo.config.provision.TenantName; import com.yahoo.container.di.config.ApplicationBundlesConfig; @@ -37,12 +36,10 @@ class GetConfigProcessor implements Runnable { private static final Logger log = Logger.getLogger(GetConfigProcessor.class.getName()); private static final String localHostName = HostName.getLocalhost(); - private static final PayloadChecksums emptyApplicationBundlesConfigChecksums = PayloadChecksums.fromPayload(Payload.from(ConfigPayload.fromInstance(new ApplicationBundlesConfig.Builder().build()))); private final JRTServerConfigRequest request; - /* True only when this request has expired its server timeout and we need to respond to the client */ private final boolean forceResponse; private final RpcServer rpcServer; @@ -75,13 +72,13 @@ class GetConfigProcessor implements Runnable { } // TODO: Increment statistics (Metrics) failed counters when requests fail - public Pair<GetConfigContext, Long> getConfig(JRTServerConfigRequest request) { + public Optional<DelayedConfig> resolveConfig(JRTServerConfigRequest request) { // Request has already been detached if ( ! request.validateParameters()) { // Error code is set in verifyParameters if parameters are not OK. log.log(Level.WARNING, "Parameters for request " + request + " did not validate: " + request.errorCode() + " : " + request.errorMessage()); respond(request); - return null; + return Optional.empty(); } Trace trace = request.getRequestTrace(); debugLog(trace, "GetConfigProcessor.run() on " + localHostName); @@ -92,13 +89,13 @@ class GetConfigProcessor implements Runnable { // fabricate an empty request to cause the sentinel to stop all running services if (rpcServer.canReturnEmptySentinelConfig() && rpcServer.allTenantsLoaded() && tenant.isEmpty() && isSentinelConfigRequest(request)) { returnEmpty(request); - return null; + return Optional.empty(); } GetConfigContext context = rpcServer.createGetConfigContext(tenant, request, trace); - if (context == null || ! context.requestHandler().hasApplication(context.applicationId(), Optional.empty())) { + if (context.isEmpty() || ! context.requestHandler().hasApplication(context.applicationId(), Optional.empty())) { handleError(request, APPLICATION_NOT_LOADED, "No application exists"); - return null; + return Optional.empty(); } logPre = TenantRepository.logPre(context.applicationId()); @@ -109,7 +106,7 @@ class GetConfigProcessor implements Runnable { if ( ! context.requestHandler().hasApplication(context.applicationId(), vespaVersion)) { handleError(request, ErrorCode.UNKNOWN_VESPA_VERSION, "Unknown Vespa version in request: " + printableVespaVersion(vespaVersion)); - return null; + return Optional.empty(); } ConfigResponse config; @@ -117,14 +114,14 @@ class GetConfigProcessor implements Runnable { config = rpcServer.resolveConfig(request, context, vespaVersion); } catch (UnknownConfigDefinitionException e) { handleError(request, ErrorCode.UNKNOWN_DEFINITION, "Unknown config definition " + request.getConfigKey()); - return null; + return Optional.empty(); } catch (UnknownConfigIdException e) { handleError(request, ErrorCode.ILLEGAL_CONFIGID, "Illegal config id " + request.getConfigKey().getConfigId()); - return null; + return Optional.empty(); } catch (Throwable e) { log.log(Level.SEVERE, "Unexpected error handling config request", e); handleError(request, ErrorCode.INTERNAL_ERROR, "Internal error " + e.getMessage()); - return null; + return Optional.empty(); } // config == null is not an error, but indicates that the config will be returned later. @@ -135,7 +132,7 @@ class GetConfigProcessor implements Runnable { && ! context.requestHandler().compatibleWith(vespaVersion, context.applicationId()) // ... with a runtime version incompatible with the deploying version ... && ! emptyApplicationBundlesConfigChecksums.matches(config.getPayloadChecksums())) { // ... and there actually are incompatible user bundles, then return no config: handleError(request, ErrorCode.INCOMPATIBLE_VESPA_VERSION, "Version " + printableVespaVersion(vespaVersion) + " is binary incompatible with the latest deployed version"); - return null; + return Optional.empty(); } // debugLog(trace, "config response before encoding:" + config.toString()); @@ -144,23 +141,24 @@ class GetConfigProcessor implements Runnable { respond(request); } else { debugLog(trace, "delaying response " + request.getShortDescription()); - return new Pair<>(context, config != null ? config.getGeneration() : 0); + return Optional.of(new DelayedConfig(context, config != null ? config.getGeneration() : 0)); } - return null; + return Optional.empty(); } @Override public void run() { - Pair<GetConfigContext, Long> delayed = getConfig(request); + Optional<DelayedConfig> delayed = resolveConfig(request); - if (delayed != null) { - rpcServer.delayResponse(request, delayed.getFirst()); - if (rpcServer.hasNewerGeneration(delayed.getFirst().applicationId(), delayed.getSecond())) { + delayed.ifPresent(d -> { + GetConfigContext context = d.context(); + if (rpcServer.hasNewerGeneration(context.applicationId(), d.generation())) // This will ensure that if the config activation train left the station while I was boarding, - // another train will immediately be scheduled. - rpcServer.configActivated(delayed.getFirst().applicationId()); - } - } + // we will resolve config again with new generation + resolveConfig(request); + else + rpcServer.delayResponse(request, context); + }); } private boolean isSentinelConfigRequest(JRTServerConfigRequest request) { @@ -194,4 +192,6 @@ class GetConfigProcessor implements Runnable { } } + private record DelayedConfig(GetConfigContext context, long generation) {} + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index be4738258d8..eee7d6ec63d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -44,11 +44,9 @@ import com.yahoo.vespa.filedistribution.FileDownloader; import com.yahoo.vespa.filedistribution.FileReceiver; import com.yahoo.vespa.filedistribution.FileReferenceData; import com.yahoo.vespa.filedistribution.FileReferenceDownload; - import java.nio.ByteBuffer; import java.time.Duration; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -186,7 +184,7 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList @Override public void run() { - log.log(Level.INFO, "Rpc server will listen on port " + spec.port()); + log.log(Level.FINE, "Rpc server will listen on port " + spec.port()); try { Acceptor acceptor = supervisor.listen(spec); isRunning = true; @@ -320,17 +318,6 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList } @Override - public void hostsUpdated(ApplicationId applicationId, Collection<String> newHosts) { - log.log(Level.FINE, () -> "Updating hosts in tenant host registry '" + hostRegistry + "' with " + newHosts); - hostRegistry.update(applicationId, newHosts); - } - - @Override - public void verifyHostsAreAvailable(ApplicationId applicationId, Collection<String> newHosts) { - hostRegistry.verifyHosts(applicationId, newHosts); - } - - @Override public void applicationRemoved(ApplicationId applicationId) { superModelRequestHandler.removeApplication(applicationId); configActivated(applicationId); @@ -350,8 +337,9 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList */ Optional<TenantName> resolveTenant(JRTServerConfigRequest request, Trace trace) { if ("*".equals(request.getConfigKey().getConfigId())) return Optional.of(ApplicationId.global().tenant()); + String hostname = request.getClientHostName(); - ApplicationId applicationId = hostRegistry.getKeyForHost(hostname); + ApplicationId applicationId = hostRegistry.getApplicationId(hostname); if (applicationId == null) { if (GetConfigProcessor.logDebug(trace)) { String message = "Did not find tenant for host '" + hostname + "', using " + TenantName.defaultName() + @@ -418,7 +406,7 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList metrics.incUnknownHostRequests(); trace.trace(TRACELEVEL, msg); log.log(Level.WARNING, msg); - return null; + return GetConfigContext.empty(); } RequestHandler handler = requestHandler.get(); ApplicationId applicationId = handler.resolveApplicationId(request.getClientHostName()); @@ -445,7 +433,6 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList log.log(Level.FINE, () -> TenantRepository.logPre(tenant) + "Tenant deleted, removing request handler and cleaning host registry"); tenants.remove(tenant); - hostRegistry.removeHostsForKey(tenant); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java index 536a446df2f..21f7354401f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java @@ -106,7 +106,7 @@ public class MultiTenantRpcAuthorizer implements RpcAuthorizer { return; // global config access ok } else { String hostname = configRequest.getClientHostName(); - ApplicationId applicationId = hostRegistry.getKeyForHost(hostname); + ApplicationId applicationId = hostRegistry.getApplicationId(hostname); if (applicationId == null) { if (isConfigKeyForSentinelConfig(configKey)) { return; // config processor will return empty sentinel config for unknown nodes diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index ffafbb8827e..06c3aa5330e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -270,8 +270,7 @@ public class SessionPreparer { // Validate after doing our own preprocessing on these two files ApplicationMetaData meta = applicationPackage.getMetaData(); InstanceName instance = meta.getApplicationId().instance(); - Tags tags = applicationPackage.getDeployment().map(new DeploymentSpecXmlReader(false)::read) - .flatMap(spec -> spec.instance(instance)) + Tags tags = applicationPackage.getDeploymentSpec().instance(instance) .map(DeploymentInstanceSpec::tags) .orElse(Tags.empty()); if (servicesXml.exists()) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 69d13bf2dea..ba09b3de365 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -372,7 +372,7 @@ public class TenantRepository { modelFactoryRegistry, configDefinitionRepo, zookeeperServerConfig.juteMaxBuffer()); - log.log(Level.INFO, "Adding tenant '" + tenantName + "'" + ", created " + created + + log.log(Level.FINE, "Adding tenant '" + tenantName + "'" + ", created " + created + ". Bootstrapping in " + Duration.between(start, clock.instant())); Tenant tenant = new Tenant(tenantName, sessionRepository, applicationRepo, created); createAndWriteTenantMetaData(tenant); @@ -401,6 +401,7 @@ public class TenantRepository { } private void notifyRemovedTenant(TenantName name) { + hostRegistry.removeHosts(name); tenantListener.onTenantDelete(name); } @@ -618,4 +619,6 @@ public class TenantRepository { public com.yahoo.vespa.curator.Curator getCurator() { return curator; } + public HostProvisionerProvider hostProvisionerProvider() { return hostProvisionerProvider; } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java index fbf14fbdb8c..6d6901136a6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -6,6 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.codegen.DefParser; @@ -55,6 +56,8 @@ public class ZKApplicationPackage extends AbstractApplicationPackage { public static final String allocatedHostsNode = "allocatedHosts"; private final ApplicationMetaData metaData; + private DeploymentSpec deploymentSpec = null; + public ZKApplicationPackage(AddFileInterface fileManager, Curator curator, Path sessionPath, int maxNodeSize) { verifyAppPath(curator, sessionPath); zkApplication = new ZKApplication(curator, sessionPath, maxNodeSize); @@ -73,6 +76,12 @@ public class ZKApplicationPackage extends AbstractApplicationPackage { return Optional.of(readAllocatedHosts()); } + @Override + public DeploymentSpec getDeploymentSpec() { + if (deploymentSpec != null) return deploymentSpec; + return deploymentSpec = parseDeploymentSpec(false); + } + /** * Reads allocated hosts at the given node. * diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 498940f8a63..253b0d7c101 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -13,7 +13,6 @@ import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Deployment; -import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NetworkPorts; @@ -38,6 +37,7 @@ import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.filedistribution.FileDirectory; import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionFactory; import com.yahoo.vespa.config.server.http.v2.PrepareResult; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.Session; @@ -79,7 +79,6 @@ import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** * @author hmusum @@ -125,21 +124,21 @@ public class ApplicationRepositoryTest { .build(); flagSource = new InMemoryFlagSource(); fileDirectory = new FileDirectory(configserverConfig); + provisioner = new MockProvisioner(); tenantRepository = new TestTenantRepository.Builder() .withClock(clock) .withConfigserverConfig(configserverConfig) .withCurator(curator) .withFileDistributionFactory(new MockFileDistributionFactory(configserverConfig)) .withFlagSource(flagSource) + .withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, configserverConfig)) .build(); tenantRepository.addTenant(TenantRepository.HOSTED_VESPA_TENANT); tenantRepository.addTenant(tenant1); tenantRepository.addTenant(tenant2); orchestrator = new OrchestratorMock(); - provisioner = new MockProvisioner(); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(provisioner) .withConfigserverConfig(configserverConfig) .withOrchestrator(orchestrator) .withLogRetriever(new MockLogRetriever()) @@ -177,7 +176,6 @@ public class ApplicationRepositoryTest { public void prepareAndActivateWithRestart() { applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(provisioner) .withConfigserverConfig(configserverConfig) .withOrchestrator(orchestrator) .withLogRetriever(new MockLogRetriever()) @@ -188,8 +186,7 @@ public class ApplicationRepositoryTest { prepareAndActivate(testAppJdiscOnly); PrepareResult result = prepareAndActivate(testAppJdiscOnlyRestart); assertTrue(result.configChangeActions().getRefeedActions().isEmpty()); - assertTrue(result.configChangeActions().getRestartActions().isEmpty()); - assertEquals(HostFilter.hostname("mytesthost2"), provisioner.lastRestartFilter()); + assertFalse(result.configChangeActions().getRestartActions().isEmpty()); } @Test @@ -197,7 +194,6 @@ public class ApplicationRepositoryTest { applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) .withOrchestrator(orchestrator) - .withProvisioner(null) .build(); prepareAndActivate(testAppJdiscOnly); @@ -285,7 +281,6 @@ public class ApplicationRepositoryTest { applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(provisioner) .withOrchestrator(orchestrator) .withClock(clock) .build(); @@ -329,20 +324,16 @@ public class ApplicationRepositoryTest { File sessionFile = new File(tenantFileSystemDirs.sessionsPath(), String.valueOf(sessionId)); assertTrue(sessionFile.exists()); - // Delete app and verify that it has been deleted from repos and provisioner and no application set exists + // Delete app and verify that it has been deleted from repos and no application set exists assertTrue(applicationRepository.delete(applicationId())); assertTrue(applicationRepository.getActiveSession(applicationId()).isEmpty()); assertEquals(Optional.empty(), sessionRepository.getRemoteSession(sessionId).applicationSet()); - assertEquals(1, provisioner.removeCount()); - assertEquals(tenant().getName(), provisioner.lastApplicationId().tenant()); - assertEquals(applicationId(), provisioner.lastApplicationId()); assertTrue(curator.exists(sessionNode)); assertEquals(Session.Status.DELETE.name(), Utf8.toString(curator.getData(sessionNode.append("sessionState")).get())); assertTrue(sessionFile.exists()); - // Deleting a non-existent application still attempts to remove resources + // Deleting a non-existent application will return false assertFalse(applicationRepository.delete(applicationId())); - assertEquals(2, provisioner.removeCount()); } { @@ -358,46 +349,9 @@ public class ApplicationRepositoryTest { // Delete app with id fooId, should not affect original app assertTrue(applicationRepository.delete(fooId)); - assertEquals(fooId, provisioner.lastApplicationId()); - assertNotNull(applicationRepository.getActiveSession(applicationId())); - - assertTrue(applicationRepository.delete(applicationId())); - } - - // If delete fails, a retry should work if the failure is transient and zookeeper state should be consistent - { - long sessionId = deployApp(testApp).sessionId(); - assertNotNull(sessionRepository.getRemoteSession(sessionId)); - assertNotNull(applicationRepository.getActiveSession(applicationId())); - assertEquals(sessionId, applicationRepository.getActiveSession(applicationId()).get().getSessionId()); - assertNotNull(applicationRepository.getApplication(applicationId())); - - provisioner.failureOnRemove(true); - try { - applicationRepository.delete(applicationId()); - fail("Should fail with RuntimeException"); - } catch (RuntimeException e) { - // ignore - } - assertNotNull(sessionRepository.getRemoteSession(sessionId)); assertNotNull(applicationRepository.getActiveSession(applicationId())); - assertEquals(sessionId, applicationRepository.getActiveSession(applicationId()).get().getSessionId()); - // Delete should work when there is no failure anymore - provisioner.failureOnRemove(false); assertTrue(applicationRepository.delete(applicationId())); - - // Session should be in state DELETE - Path sessionNode = sessionRepository.getSessionPath(sessionId); - assertEquals(Session.Status.DELETE.name(), Utf8.toString(curator.getData(sessionNode.append("sessionState")).get())); - assertNotNull(sessionRepository.getRemoteSession(sessionId)); // session still exists - assertTrue(applicationRepository.getActiveSession(applicationId()).isEmpty()); // but it is not active - try { - applicationRepository.getApplication(applicationId()); - fail("Should fail with NotFoundException, application should not exist"); - } catch (NotFoundException e) { - // ignore - } } } @@ -522,7 +476,6 @@ public class ApplicationRepositoryTest { MockMetric actual = new MockMetric(); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(provisioner) .withOrchestrator(orchestrator) .withMetric(actual) .withClock(new ManualClock()) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockProvisioner.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockProvisioner.java index 0ba3a6d883c..8effd9b6dfe 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockProvisioner.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockProvisioner.java @@ -22,15 +22,7 @@ import java.util.List; */ public class MockProvisioner implements Provisioner { - private boolean activated = false; - private int removeCount = 0; - private boolean restarted = false; - private ApplicationId lastApplicationId; - private Collection<HostSpec> lastHosts; - private HostFilter lastRestartFilter; - private boolean transientFailureOnPrepare = false; - private boolean failureOnRemove = false; private HostProvisioner hostProvisioner = null; public MockProvisioner hostProvisioner(HostProvisioner hostProvisioner) { @@ -43,10 +35,6 @@ public class MockProvisioner implements Provisioner { return this; } - public void failureOnRemove(boolean failureOnRemove) { - this.failureOnRemove = failureOnRemove; - } - @Override public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { if (hostProvisioner != null) { @@ -60,25 +48,14 @@ public class MockProvisioner implements Provisioner { @Override public void activate(Collection<HostSpec> hosts, ActivationContext context, ApplicationTransaction transaction) { - activated = true; - lastApplicationId = transaction.application(); - lastHosts = hosts; } @Override public void remove(ApplicationTransaction transaction) { - if (failureOnRemove) - throw new IllegalStateException("Unable to remove " + transaction.application()); - - removeCount++; - lastApplicationId = transaction.application(); } @Override public void restart(ApplicationId application, HostFilter filter) { - restarted = true; - lastApplicationId = application; - lastRestartFilter = filter; } @Override @@ -86,28 +63,4 @@ public class MockProvisioner implements Provisioner { return new ProvisionLock(application, () -> {}); } - public Collection<HostSpec> lastHosts() { - return lastHosts; - } - - public boolean activated() { - return activated; - } - - public int removeCount() { - return removeCount; - } - - public boolean restarted() { - return restarted; - } - - public ApplicationId lastApplicationId() { - return lastApplicationId; - } - - public HostFilter lastRestartFilter() { - return lastRestartFilter; - } - } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java index 85e64b4a32d..728f3e8510f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java @@ -203,15 +203,6 @@ public class TenantApplicationsTest { } @Override - public void hostsUpdated(ApplicationId applicationId, Collection<String> newHosts) { - tenantHosts.put(applicationId.tenant().value(), newHosts); - } - - @Override - public void verifyHostsAreAvailable(ApplicationId applicationId, Collection<String> newHosts) { - } - - @Override public void applicationRemoved(ApplicationId applicationId) { removed.incrementAndGet(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index ab527833803..c716219f86b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -296,7 +296,8 @@ public class DeployTester { .withZone(zone) .withFlagSource(flagSource); - if (configserverConfig.hostedVespa()) builder.withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, true)); + if (configserverConfig.hostedVespa()) + builder.withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, configserverConfig)); TenantRepository tenantRepository = builder.build(); tenantRepository.addTenant(tenantName); @@ -306,7 +307,6 @@ public class DeployTester { .withConfigserverConfig(configserverConfig) .withOrchestrator(new OrchestratorMock()) .withClock(clock) - .withProvisioner(provisioner) .withConfigConvergenceChecker(configConvergenceChecker) .withFlagSource(flagSource) .build(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/host/HostRegistryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/host/HostRegistryTest.java index df00d28134f..646017a498e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/host/HostRegistryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/host/HostRegistryTest.java @@ -24,23 +24,23 @@ public class HostRegistryTest { @Test public void old_hosts_are_removed() { HostRegistry reg = new HostRegistry(); - assertNull(reg.getKeyForHost("foo.com")); + assertNull(reg.getApplicationId("foo.com")); reg.update(foo, List.of("foo.com", "bar.com", "baz.com")); assertGetKey(reg, "foo.com", foo); assertGetKey(reg, "bar.com", foo); assertGetKey(reg, "baz.com", foo); assertEquals(3, reg.getAllHosts().size()); reg.update(foo, List.of("bar.com", "baz.com")); - assertNull(reg.getKeyForHost("foo.com")); + assertNull(reg.getApplicationId("foo.com")); assertGetKey(reg, "bar.com", foo); assertGetKey(reg, "baz.com", foo); assertEquals(2, reg.getAllHosts().size()); assertTrue(reg.getAllHosts().containsAll(List.of("bar.com", "baz.com"))); - reg.removeHostsForKey(foo); + reg.removeHosts(foo); assertTrue(reg.getAllHosts().isEmpty()); - assertNull(reg.getKeyForHost("foo.com")); - assertNull(reg.getKeyForHost("bar.com")); + assertNull(reg.getApplicationId("foo.com")); + assertNull(reg.getApplicationId("bar.com")); } @Test @@ -74,9 +74,9 @@ public class HostRegistryTest { HostRegistry reg = new HostRegistry(); List<String> hosts = new ArrayList<>(List.of("foo.com", "bar.com", "baz.com")); reg.update(foo, hosts); - assertEquals(3, reg.getHostsForKey(foo).size()); + assertEquals(3, reg.getHosts(foo).size()); hosts.remove(2); - assertEquals(3, reg.getHostsForKey(foo).size()); + assertEquals(3, reg.getHosts(foo).size()); } @Test @@ -90,8 +90,8 @@ public class HostRegistryTest { } private void assertGetKey(HostRegistry reg, String host, ApplicationId expectedKey) { - assertNotNull(reg.getKeyForHost(host)); - assertEquals(expectedKey, reg.getKeyForHost(host)); + assertNotNull(reg.getApplicationId(host)); + assertEquals(expectedKey, reg.getApplicationId(host)); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java index 279f3a237e8..6cfdab1257f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.TenantRepository; @@ -16,7 +15,6 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.File; import java.io.IOException; import java.util.Collections; @@ -59,7 +57,6 @@ public class HttpGetConfigHandlerTest { tenantRepository.addTenant(tenant); ApplicationRepository applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withConfigserverConfig(configserverConfig) .build(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java index 520b4d0edc5..79881d07b25 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java @@ -8,7 +8,6 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.HttpListConfigsHandler.ListConfigsResponse; import com.yahoo.vespa.config.server.session.PrepareParams; @@ -18,7 +17,6 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.File; import java.io.IOException; import java.util.HashSet; @@ -64,7 +62,6 @@ public class HttpListConfigsHandlerTest { tenantRepository.addTenant(tenant); ApplicationRepository applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withConfigserverConfig(configserverConfig) .build(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java index 052f39c9e1f..60ee3299de5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java @@ -9,7 +9,6 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Response; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.ContentHandlerTestBase; import com.yahoo.vespa.config.server.session.PrepareParams; @@ -21,7 +20,6 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.File; import java.io.IOException; @@ -63,7 +61,6 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withConfigserverConfig(configserverConfig) .build(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index c270b4559f9..e8c4d819c31 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -87,9 +87,7 @@ import static com.yahoo.vespa.config.server.http.v2.ApplicationHandler.HttpServi import static com.yahoo.yolean.Exceptions.uncheck; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -136,14 +134,13 @@ public class ApplicationHandlerTest { .withClock(clock) .withConfigserverConfig(configserverConfig) .withFileDistributionFactory(new MockFileDistributionFactory(configserverConfig)) - .withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, false)) + .withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, configserverConfig)) .withModelFactoryRegistry(new ModelFactoryRegistry(modelFactories)) .build(); tenantRepository.addTenant(mytenantName); orchestrator = new OrchestratorMock(); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(provisioner) .withOrchestrator(orchestrator) .withClock(clock) .withTesterClient(testerClient) @@ -350,11 +347,9 @@ public class ApplicationHandlerTest { @Test public void testRestart() throws Exception { - applicationRepository.deploy(testApp, prepareParams(applicationId)); - assertFalse(provisioner.restarted()); + var result = applicationRepository.deploy(testApp, prepareParams(applicationId)); + assertTrue(result.configChangeActions().getRestartActions().isEmpty()); restart(applicationId, Zone.defaultZone()); - assertTrue(provisioner.restarted()); - assertEquals(applicationId, provisioner.lastApplicationId()); } @Test @@ -378,7 +373,6 @@ public class ApplicationHandlerTest { HttpProxy mockHttpProxy = mock(HttpProxy.class); ApplicationRepository applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withHostProvisionerProvider(HostProvisionerProvider.empty()) .withOrchestrator(orchestrator) .withTesterClient(testerClient) .withHttpProxy(mockHttpProxy) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java index fbc5e87c329..ba1d69c13dd 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java @@ -11,7 +11,6 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Response; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; @@ -22,7 +21,6 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.File; import java.io.IOException; @@ -58,7 +56,6 @@ public class HostHandlerTest { tenantRepository.addTenant(mytenant); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withConfigserverConfig(configserverConfig) .build(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java index 9aae64cb884..401bd1ae55b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpConfigRequest; @@ -23,10 +22,10 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.File; import java.io.IOException; import java.util.Collections; +import java.util.Map; import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; import static com.yahoo.jdisc.Response.Status.NOT_FOUND; @@ -60,15 +59,13 @@ public class HttpGetConfigHandlerTest { .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) .build(); - MockProvisioner provisioner = new MockProvisioner(); TenantRepository tenantRepository = new TestTenantRepository.Builder() .withConfigserverConfig(configserverConfig) - .withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, false)) + .withHostProvisionerProvider(HostProvisionerProvider.empty()) .build(); tenantRepository.addTenant(tenant); ApplicationRepository applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(provisioner) .withOrchestrator(new OrchestratorMock()) .withConfigserverConfig(configserverConfig) .build(); @@ -135,6 +132,20 @@ public class HttpGetConfigHandlerTest { assertTrue(renderedString, renderedString.startsWith(expected)); } + @Test + public void require_that_required_generation_property_works() throws IOException { + HttpRequest request = HttpRequest.createTestRequest(configUri, GET, null, Map.of("requiredGeneration", "2")); + HttpResponse response = handler.handle(request); + String renderedString = SessionHandlerTest.getRenderedString(response); + assertTrue(renderedString, renderedString.startsWith(expected)); + + request = HttpRequest.createTestRequest(configUri, GET, null, Map.of("requiredGeneration", "3")); + response = handler.handle(request); + assertEquals(412, response.getStatus()); + renderedString = SessionHandlerTest.getRenderedString(response); + assertEquals("{\"error-code\":\"PRECONDITION_FAILED\",\"message\":\"Config for required generation 3 could not be found.\"}", renderedString); + } + private PrepareParams prepareParams() { return new PrepareParams.Builder().applicationId(applicationId).build(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java index 2ee1064f614..3762e52ae62 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java @@ -11,7 +11,6 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; @@ -23,7 +22,6 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.File; import java.io.IOException; import java.util.HashSet; @@ -71,7 +69,6 @@ public class HttpListConfigsHandlerTest { tenantRepository.addTenant(tenant); ApplicationRepository applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withConfigserverConfig(configserverConfig) .build(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index 1c71ef0b7fb..d7e6273352b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -19,6 +19,7 @@ import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.model.TestModelFactory; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.config.server.tenant.Tenant; @@ -74,11 +75,11 @@ public class SessionActiveHandlerTest { TenantRepository tenantRepository = new TestTenantRepository.Builder() .withConfigserverConfig(configserverConfig) .withModelFactoryRegistry(new ModelFactoryRegistry(List.of((modelFactory)))) + .withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, configserverConfig)) .build(); tenantRepository.addTenant(tenantName); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(provisioner) .withOrchestrator(new OrchestratorMock()) .withClock(clock) .withConfigserverConfig(configserverConfig) @@ -164,8 +165,6 @@ public class SessionActiveHandlerTest { "/environment/" + "prod" + "/region/" + "default" + "/instance/" + "default")); - assertTrue(provisioner.activated()); - assertEquals(1, provisioner.lastHosts().size()); } private SessionActiveHandler createHandler() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java index 7c2e0be0c3a..b17f80fd510 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java @@ -11,7 +11,6 @@ import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.ContentHandlerTestBase; import com.yahoo.vespa.config.server.http.SessionHandlerTest; @@ -24,7 +23,6 @@ import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; @@ -63,7 +61,6 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { ApplicationRepository applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withConfigserverConfig(configserverConfig) .build(); @@ -186,7 +183,6 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { SessionContentHandler.testContext(), new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withClock(Clock.systemUTC()) .build() diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java index 2e86f5e0538..04531fbb2e0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java @@ -1,15 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import ai.vespa.http.HttpURL.Path; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import ai.vespa.http.HttpURL.Path; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.application.CompressedApplicationInputStreamTest; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.HttpErrorResponse; @@ -23,7 +22,6 @@ import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; @@ -84,7 +82,6 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { .build(); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .build(); tenantRepository.addTenant(tenant); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java index de6073bb1ea..765523177a9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java @@ -16,7 +16,6 @@ import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.HttpErrorResponse; @@ -30,7 +29,6 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -82,7 +80,6 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { tenantRepository.addTenant(tenant); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withClock(clock) .withConfigserverConfig(configserverConfig) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java index b8bd35a564a..b39050250f9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java @@ -10,7 +10,6 @@ import com.yahoo.container.jdisc.HttpRequestBuilder; import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.restapi.RestApiTestDriver; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.TenantRepository; @@ -20,7 +19,6 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -58,7 +56,6 @@ public class TenantHandlerTest { .build(); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withConfigserverConfig(configserverConfig) .build(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java index a2dc0216b72..5fb92e1f66f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java @@ -41,13 +41,12 @@ class MaintainerTester { .build(); tenantRepository = new TestTenantRepository.Builder() .withClock(clock) - .withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, true)) + .withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, configserverConfig)) .withConfigserverConfig(configserverConfig) .withModelFactoryRegistry(new ModelFactoryRegistry(List.of(new DeployTester.CountingModelFactory(clock)))) .build(); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(provisioner) .withOrchestrator(new OrchestratorMock()) .withLogRetriever(new MockLogRetriever()) .withClock(clock) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java index 8607fc0e2dc..9190cbc0d8a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java @@ -38,10 +38,10 @@ import java.io.File; import java.io.IOException; import java.util.Optional; +import static com.yahoo.vespa.config.server.rpc.RpcServer.ChunkedFileReceiver.createMetaRequest; import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.gzip; import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.lz4; import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.compressed; -import static com.yahoo.vespa.config.server.rpc.RpcServer.ChunkedFileReceiver.createMetaRequest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -84,7 +84,7 @@ public class RpcServerTest { public void testEmptySentinelConfigWhenAppDeletedOnHostedVespa() throws IOException, InterruptedException { ConfigserverConfig.Builder configBuilder = new ConfigserverConfig.Builder().canReturnEmptySentinelConfig(true); try (RpcTester tester = new RpcTester(applicationId, temporaryFolder, configBuilder)) { - tester.rpcServer().onTenantDelete(tenantName); + tester.hostRegistry.removeHosts(applicationId); tester.rpcServer().onTenantsLoaded(); JRTClientConfigRequest clientReq = createSentinelRequest(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java index b29edd480ad..8770970308a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java @@ -12,7 +12,6 @@ import com.yahoo.jrt.Transport; import com.yahoo.test.ManualClock; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.MemoryGenerationCounter; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.PortRangeAllocator; import com.yahoo.vespa.config.server.SuperModelManager; import com.yahoo.vespa.config.server.SuperModelRequestHandler; @@ -55,7 +54,7 @@ public class RpcTester implements AutoCloseable { private final ApplicationId applicationId; private final TenantName tenantName; private final TenantRepository tenantRepository; - private final HostRegistry hostRegistry = new HostRegistry(); + final HostRegistry hostRegistry = new HostRegistry(); private final ApplicationRepository applicationRepository; private final List<Integer> allocatedPorts = new ArrayList<>(); @@ -87,7 +86,6 @@ public class RpcTester implements AutoCloseable { applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) .withConfigserverConfig(configserverConfig) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .build(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index c7113bbf803..cc6cd4d86e9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -335,7 +335,9 @@ public class SessionPreparerTest { @Test(expected = LoadBalancerServiceException.class) public void require_that_conflict_is_returned_when_creating_load_balancer_fails() throws IOException { - preparer = createPreparer(HostProvisionerProvider.withProvisioner(new MockProvisioner().transientFailureOnPrepare(), true)); + var configserverConfig = new ConfigserverConfig.Builder().hostedVespa(true).build(); + MockProvisioner provisioner = new MockProvisioner().transientFailureOnPrepare(); + preparer = createPreparer(HostProvisionerProvider.withProvisioner(provisioner, configserverConfig)); var params = new PrepareParams.Builder().applicationId(applicationId("test")).build(); prepare(new File("src/test/resources/deploy/hosted-app"), params); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java index a5360fbc01c..83ada4122c2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java @@ -19,7 +19,6 @@ import com.yahoo.io.reader.NamedReader; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionFactory; @@ -103,7 +102,6 @@ public class SessionRepositoryTest { tenantRepository.addTenant(SessionRepositoryTest.tenantName); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withFlagSource(flagSource) .build(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java index 823466603b1..9af1bbb875e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java @@ -13,7 +13,6 @@ import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.server.ConfigServerDB; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.MockSecretStore; import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.TestConfigDefinitionRepo; @@ -219,7 +218,7 @@ public class TenantRepositoryTest { flagSource, new InThreadExecutorService(), new MockSecretStore(), - HostProvisionerProvider.withProvisioner(new MockProvisioner(), false), + HostProvisionerProvider.empty(), configserverConfig, new ConfigServerDB(configserverConfig), Zone.defaultZone(), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index 8f11e171ebe..f3a2c42852f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -101,7 +101,9 @@ public class ZKApplicationPackageTest { assertEquals("6.0.1", readInfo.getHosts().iterator().next().version().get().toString()); assertEquals(dockerImage, readInfo.getHosts().iterator().next().dockerImageRepo().get().asString()); assertTrue(zkApp.getDeployment().isPresent()); + assertFalse(zkApp.getDeploymentSpec().isEmpty()); assertEquals("mydisc", DeploymentSpec.fromXml(zkApp.getDeployment().get()).requireInstance("default").globalServiceId().get()); + assertEquals("mydisc", zkApp.getDeploymentSpec().requireInstance("default").globalServiceId().get()); } private void feed(com.yahoo.vespa.curator.Curator zk, File dirToFeed) throws IOException { diff --git a/configutil/CMakeLists.txt b/configutil/CMakeLists.txt index 4ec9ad13648..a1d699ae009 100644 --- a/configutil/CMakeLists.txt +++ b/configutil/CMakeLists.txt @@ -2,7 +2,6 @@ vespa_define_module( DEPENDS vespadefaults - fastos config_cloudconfig vbench vespalib diff --git a/container-core/src/main/java/com/yahoo/container/core/config/PlatformBundleLoader.java b/container-core/src/main/java/com/yahoo/container/core/config/PlatformBundleLoader.java index 6ed0a876f4e..a84436083ff 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/PlatformBundleLoader.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/PlatformBundleLoader.java @@ -7,6 +7,7 @@ import org.osgi.framework.Bundle; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -41,7 +42,7 @@ public class PlatformBundleLoader { public void useBundles(List<String> bundlePaths) { if (hasLoadedBundles) { - log.fine(() -> "Platform bundles have already been installed." + + log.log(Level.FINE, () -> "Platform bundles have already been installed." + "\nInstalled bundles: " + installedBundles + "\nGiven files: " + bundlePaths); return; @@ -65,9 +66,9 @@ public class PlatformBundleLoader { } private List<Bundle> installBundleFromDisk(String bundlePath) { - log.info("Installing bundle from disk: " + bundlePath); + log.log(Level.FINE, "Installing bundle from disk: " + bundlePath); List<Bundle> bundles = installer.installBundles(bundlePath, osgi); - log.fine("Installed " + bundles.size() + " bundles for file " + bundlePath); + log.log(Level.FINE, "Installed " + bundles.size() + " bundles for file " + bundlePath); return bundles; } diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java index 2238abc584e..b9a6d8d9462 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java @@ -6,6 +6,7 @@ import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.core.VipStatusConfig; import com.yahoo.container.jdisc.state.StateMonitor; import com.yahoo.jdisc.Metric; +import com.yahoo.metrics.ContainerMetrics; import java.util.Map; import java.util.stream.Collectors; @@ -119,7 +120,7 @@ public class VipStatus { else if (healthState.status() == StateMonitor.Status.up) healthState.status(StateMonitor.Status.down); - metric.set("in_service", currentlyInRotation ? 1 : 0, metric.createContext(Map.of())); + metric.set(ContainerMetrics.IN_SERVICE.baseName(), currentlyInRotation ? 1 : 0, metric.createContext(Map.of())); } } diff --git a/container-core/src/main/java/com/yahoo/metrics/ConfigServerMetrics.java b/container-core/src/main/java/com/yahoo/metrics/ConfigServerMetrics.java new file mode 100644 index 00000000000..e501fa5e832 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/ConfigServerMetrics.java @@ -0,0 +1,60 @@ +package com.yahoo.metrics; + +/** + * @author yngveaasheim + */ +public enum ConfigServerMetrics implements VespaMetrics { + + REQUESTS("configserver.requests", Unit.REQUEST, "Number of requests processed"), + FAILED_REQUESTS("configserver.failedRequests", Unit.REQUEST, "Number of requests that failed"), + LATENCY("configserver.latency", Unit.MILLISECOND, "Time to complete requests"), + CACHE_CONFIG_ELEMS("configserver.cacheConfigElems", Unit.ITEM, "Time to complete requests"), + CACHE_CHECKSUM_ELEMS("configserver.cacheChecksumElems", Unit.ITEM, "Number of checksum elements in the cache"), + HOSTS("configserver.hosts", Unit.NODE, "The number of nodes being served configuration from the config server cluster"), + TENANTS("configserver.tenants", Unit.INSTANCE, "The number of tenants being served configuration from the config server cluster"), + APPLICATIONS("configserver.applications", Unit.INSTANCE, "The number of applications being served configuration from the config server cluster"), + DELAYED_RESPONSES("configserver.delayedResponses", Unit.RESPONSE, "Number of delayed responses"), + SESSION_CHANGE_ERRORS("configserver.sessionChangeErrors", Unit.SESSION, "Number of session change errors"), + UNKNOWN_HOST_REQUEST("configserver.unknownHostRequests", Unit.REQUEST, "Config requests from unknown hosts"), + NEW_SESSIONS("configserver.newSessions", Unit.SESSION, "New config sessions"), + PREPARED_SESSIONS("configserver.preparedSessions", Unit.SESSION, "Prepared config sessions"), + ACTIVE_SESSIONS("configserver.activeSessions", Unit.SESSION, "Active config sessions"), + INACTIVE_SESSIONS("configserver.inactiveSessions", Unit.SESSION, "Inactive config sessions"), + ADDED_SESSIONS("configserver.addedSessions", Unit.SESSION, "Added config sessions"), + REMOVED_SESSIONS("configserver.removedSessions", Unit.SESSION, "Removed config sessions"), + RPC_SERVER_WORK_QUEUE_SIZE("configserver.rpcServerWorkQueueSize", Unit.ITEM, "Number of elements in the RPC server work queue"), + + // ZooKeeper related metrics + ZK_CONNECTIONS_LOST("configserver.zkConnectionLost", Unit.CONNECTION, "Number of ZooKeeper connections lost"), + ZK_RECONNECTED("configserver.zkReconnected", Unit.CONNECTION, "Number of ZooKeeper reconnections"), + ZK_CONNECTED("configserver.zkConnected", Unit.NODE, "Number of ZooKeeper nodes connected"), + ZK_SUSPENDED("configserver.zkSuspended", Unit.NODE, "Number of ZooKeeper nodes suspended"), + ZK_Z_NODES("configserver.zkZNodes", Unit.NODE, "Number of ZooKeeper nodes present"), + ZK_AVG_LATENCY("configserver.zkAvgLatency", Unit.MILLISECOND, "Average latency for ZooKeeper requests"), // TODO: Confirm metric name + ZK_MAX_LATENCY("configserver.zkMaxLatency", Unit.MILLISECOND, "Max latency for ZooKeeper requests"), + ZK_CONNECTIONS("configserver.zkConnections", Unit.CONNECTION, "Number of ZooKeeper connections"), + ZK_OUTSTANDING_REQUESTS("configserver.zkOutstandingRequests", Unit.REQUEST, "Number of ZooKeeper requests in flight"); + + private final String name; + private final Unit unit; + private final String description; + + ConfigServerMetrics(String name, Unit unit, String description) { + this.name = name; + this.unit = unit; + this.description = description; + } + + public String baseName() { + return name; + } + + public Unit unit() { + return unit; + } + + public String description() { + return description; + } + +} diff --git a/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java b/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java index 06285ecfba8..ed1d6f7a001 100644 --- a/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java +++ b/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java @@ -11,6 +11,9 @@ public enum ContainerMetrics implements VespaMetrics { HTTP_STATUS_4XX("http.status.4xx", Unit.RESPONSE, "Number of responses with a 4xx status"), HTTP_STATUS_5XX("http.status.5xx", Unit.RESPONSE, "Number of responses with a 5xx status"), + APPLICATION_GENERATION("application_generation", Unit.VERSION, "The currently live application config generation (aka session id)"), + IN_SERVICE("in_service", Unit.BINARY, "This will have the value 1 if the node is in service, 0 if not."), + JDISC_GC_COUNT("jdisc.gc.count", Unit.OPERATION, "Number of JVM garbage collections done"), JDISC_GC_MS("jdisc.gc.ms", Unit.MILLISECOND, "Time spent in JVM garbage collection"), JDISC_JVM("jdisc.jvm", Unit.VERSION, "JVM runtime version"), @@ -182,7 +185,15 @@ public enum ContainerMetrics implements VespaMetrics { CLUSTER_CONTROLLER_RESOURCE_USAGE_MAX_DISK_UTILIZATION("cluster-controller.resource_usage.max_disk_utilization", Unit.FRACTION, "Current disk space utilisation, per content node"), CLUSTER_CONTROLLER_RESOURCE_USAGE_MEMORY_LIMIT("cluster-controller.resource_usage.memory_limit", Unit.FRACTION, "Disk space limit as a fraction of available disk space"), CLUSTER_CONTROLLER_RESOURCE_USAGE_DISK_LIMIT("cluster-controller.resource_usage.disk_limit", Unit.FRACTION, "Memory space limit as a fraction of available memory"), - CLUSTER_CONTROLLER_REINDEXING_PROGRESS("reindexing.progress", Unit.FRACTION, "Re-indexing progress"); + CLUSTER_CONTROLLER_REINDEXING_PROGRESS("reindexing.progress", Unit.FRACTION, "Re-indexing progress"), + + // Java (JRT) TLS metrics + JRT_TRANSPORT_TLS_CERTIFICATE_VERIFICATION_FAILURES("jrt.transport.tls-certificate-verification-failures", Unit.FAILURE, "TLS certificate verification failures"), + JRT_TRANSPORT_PEER_AUTHORIZATION_FAILURES("jrt.transport.peer-authorization-failures", Unit.FAILURE, "TLS peer authorization failures"), + JRT_TRANSPORT_SERVER_TLS_CONNECIONTS_ESTABLISHED("jrt.transport.server.tls-connections-established", Unit.CONNECTION, "TLS server connections established"), + JRT_TRANSPORT_CLIENT_TLS_CONNECTIONS_ESTABLISHED("jrt.transport.client.tls-connections-established", Unit.CONNECTION, "TLS client connections established"), + JRT_TRANSPORT_SERVER_UNENCRYPTED_CONNECTIONS_ESTABLISHED("jrt.transport.server.unencrypted-connections-established", Unit.CONNECTION, "Unencrypted server connections established"), + JRT_TRANSPORT_CLIENT_UNENCRYPTED_CONNECTIONS_ESTABLISHED("jrt.transport.client.unencrypted-connections-established", Unit.CONNECTION, "Unencrypted client connections established"); private final String name; diff --git a/container-core/src/main/java/com/yahoo/metrics/HostedNodeAdminMetrics.java b/container-core/src/main/java/com/yahoo/metrics/HostedNodeAdminMetrics.java new file mode 100644 index 00000000000..5624f1f92e3 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/HostedNodeAdminMetrics.java @@ -0,0 +1,61 @@ +package com.yahoo.metrics; + +/** + * @author yngveaasheim + */ + +// TODO: Move to hosted repo. +public enum HostedNodeAdminMetrics implements VespaMetrics { + + // System metrics + CPU_UTIL("cpu.util", Unit.FRACTION, "CPU utilisation"), + CPU_SYS_UTIL("cpu.sys.util", Unit.FRACTION, "System CPU utilisation"), + CPU_THROTTLED_TIME("cpu.throttled_time.rate", Unit.FRACTION, "Part of the time CPU is exhausted (CPU throttling enforced)"), + CPU_THROTTLED_CPU_TIME("cpu.throttled_cpu_time.rate", Unit.FRACTION, "Part of the time CPU is exhausted (CPU throttling enforced)"), + CPU_VCPUS("cpu.vcpus", Unit.ITEM, "Number of virtual CPU threads allocation to the node"), + DISK_LIMIT("disk.limit", Unit.BYTE, "Amount of disk space available on the node"), + DISK_USED("disk.used", Unit.BYTE, "Amount of disk space used by the node"), + DISK_UTIL("disk.util", Unit.FRACTION, "Disk space utilisation"), + MEM_LIMIT("mem.limit", Unit.BYTE, "Amount of memory available on the node"), + MEM_USED("mem.used", Unit.BYTE, "Amount of memory used by the node"), + MEM_UTIL("mem.util", Unit.FRACTION, "Memory utilisation"), + MEM_TOTAL_USED("mem_total.used", Unit.BYTE, "Total amount of memory used by the node, including OS buffer caches"), + MEM_TOTAL_UTIL("mem_total.util", Unit.FRACTION, "Total memory utilisation"), + GPU_UTIL("gpu.util", Unit.FRACTION, "GPU utilisation"), + GPU_MEM_USED("gpu.memory.used", Unit.BYTE, "GPU memory used"), + GPU_MEM_TOTAL("gpu.memory.total", Unit.BYTE, "GPU memory available"), + + + // Network metrics + NET_IN_BYTES("net.in.bytes", Unit.BYTE, "Network bytes received (rxBytes) (COUNT metric)"), + NET_IN_ERROR("net.in.errors", Unit.FAILURE, "Network receive errors (rxErrors)"), + NET_IN_DROPPED("net.in.dropped", Unit.PACKET, "Inbound network packets dropped (rxDropped)"), + NET_OUT_BYTES("net.in.bytes", Unit.BYTE, "Network bytes sent (txBytes) (COUNT metric)"), + NET_OUT_ERROR("net.in.errors", Unit.FAILURE, "Network send errors (txErrors)"), + NET_OUT_DROPPED("net.in.dropped", Unit.PACKET, "Outbound network packets dropped (txDropped)"), + BANDWIDTH_LIMIT("bandwidth.limit", Unit.BYTE_PER_SECOND, "Available network bandwidth"); + + private final String name; + private final Unit unit; + private final String description; + + HostedNodeAdminMetrics(String name, Unit unit, String description) { + this.name = name; + this.unit = unit; + this.description = description; + } + + public String baseName() { + return name; + } + + public Unit unit() { + return unit; + } + + public String description() { + return description; + } + +} + diff --git a/container-core/src/main/java/com/yahoo/metrics/LogdMetrics.java b/container-core/src/main/java/com/yahoo/metrics/LogdMetrics.java new file mode 100644 index 00000000000..3dae4283b9f --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/LogdMetrics.java @@ -0,0 +1,33 @@ +package com.yahoo.metrics; + +/** + * @author yngveaasheim + */ +public enum LogdMetrics implements VespaMetrics { + + LOGD_PROCESSED_LINES("logd.processed.lines", Unit.ITEM, "Number of log lines processed"); + + private final String name; + private final Unit unit; + private final String description; + + LogdMetrics(String name, Unit unit, String description) { + this.name = name; + this.unit = unit; + this.description = description; + } + + public String baseName() { + return name; + } + + public Unit unit() { + return unit; + } + + public String description() { + return description; + } + +} + diff --git a/container-core/src/main/java/com/yahoo/metrics/NodeAdminMetrics.java b/container-core/src/main/java/com/yahoo/metrics/NodeAdminMetrics.java new file mode 100644 index 00000000000..004a226f825 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/NodeAdminMetrics.java @@ -0,0 +1,35 @@ +package com.yahoo.metrics; + +/** + * @author yngveaasheim + */ +public enum NodeAdminMetrics implements VespaMetrics { + + ENDPOINT_CERTIFICATE_EXPIRY_SECONDS("endpoint.certificate.expiry.seconds", Unit.SECOND, "Time until node endpoint certificate expires"), + NODE_CERTIFICATE_EXPIRY_SECONDS("node-certificate.expiry.seconds", Unit.SECOND, "Time until node certificate expires"); + + + private final String name; + private final Unit unit; + private final String description; + + NodeAdminMetrics(String name, Unit unit, String description) { + this.name = name; + this.unit = unit; + this.description = description; + } + + public String baseName() { + return name; + } + + public Unit unit() { + return unit; + } + + public String description() { + return description; + } + +} + diff --git a/container-core/src/main/java/com/yahoo/metrics/RoutingLayerMetrics.java b/container-core/src/main/java/com/yahoo/metrics/RoutingLayerMetrics.java new file mode 100644 index 00000000000..773afae00ba --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/RoutingLayerMetrics.java @@ -0,0 +1,34 @@ +package com.yahoo.metrics; + +/** + * @author yngveaasheim + */ + +// Internal hosted Vespa only TODO: Move to a better place +public enum RoutingLayerMetrics implements VespaMetrics { + + WORKER_CONNECTIONS("worker.connections", Unit.CONNECTION, "Yahoo! Internal: Number of connections for the routing worker having most connections per node"); + + private final String name; + private final Unit unit; + private final String description; + + RoutingLayerMetrics(String name, Unit unit, String description) { + this.name = name; + this.unit = unit; + this.description = description; + } + + public String baseName() { + return name; + } + + public Unit unit() { + return unit; + } + + public String description() { + return description; + } + +} diff --git a/container-core/src/main/java/com/yahoo/metrics/SearchNodeMetrics.java b/container-core/src/main/java/com/yahoo/metrics/SearchNodeMetrics.java index b5f1406ca4c..c39054c878c 100644 --- a/container-core/src/main/java/com/yahoo/metrics/SearchNodeMetrics.java +++ b/container-core/src/main/java/com/yahoo/metrics/SearchNodeMetrics.java @@ -7,6 +7,8 @@ import java.util.List; */ public enum SearchNodeMetrics implements VespaMetrics { + CONTENT_PROTON_CONFIG_GENERATION("content.proton.config.generation", Unit.VERSION, "The oldest config generation used by this search node"), + CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_TOTAL("content.proton.documentdb.documents.total", Unit.DOCUMENT, "The total number of documents in this documents db (ready + not-ready)"), CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_READY("content.proton.documentdb.documents.ready", Unit.DOCUMENT, "The number of ready documents in this document db"), CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_ACTIVE("content.proton.documentdb.documents.active", Unit.DOCUMENT, "The number of active / searchable documents in this document db"), @@ -15,7 +17,11 @@ public enum SearchNodeMetrics implements VespaMetrics { CONTENT_PROTON_DOCUMENTDB_INDEX_DOCS_IN_MEMORY("content.proton.documentdb.index.docs_in_memory", Unit.DOCUMENT, "Number of documents in memory index"), CONTENT_PROTON_DOCUMENTDB_DISK_USAGE("content.proton.documentdb.disk_usage", Unit.BYTE, "The total disk usage (in bytes) for this document db"), CONTENT_PROTON_DOCUMENTDB_MEMORY_USAGE_ALLOCATED_BYTES("content.proton.documentdb.memory_usage.allocated_bytes", Unit.BYTE, "The number of allocated bytes"), + CONTENT_PROTON_DOCUMENTDB_MEMORY_USAGE_DEAD_BYTES("content.proton.documentdb.memory_usage.dead_bytes", Unit.BYTE, "The number of dead bytes (<= used_bytes)"), + CONTENT_PROTON_DOCUMENTDB_MEMORY_USAGE_ONHOLD_BYTES("content.proton.documentdb.memory_usage.onhold_bytes", Unit.BYTE, "The number of bytes on hold"), + CONTENT_PROTON_DOCUMENTDB_MEMORY_USAGE_USED_BYTES("content.proton.documentdb.memory_usage.used_bytes", Unit.BYTE, "The number of used bytes (<= allocated_bytes)"), CONTENT_PROTON_DOCUMENTDB_HEART_BEAT_AGE("content.proton.documentdb.heart_beat_age", Unit.SECOND, "How long ago (in seconds) heart beat maintenace job was run"), + CONTENT_PROTON_DOCSUM_COUNT("content.proton.docsum.count", Unit.REQUEST, "Docsum requests handled"), CONTENT_PROTON_DOCSUM_DOCS("content.proton.docsum.docs", Unit.DOCUMENT, "Total docsums returned"), CONTENT_PROTON_DOCSUM_LATENCY("content.proton.docsum.latency", Unit.MILLISECOND, "Docsum request latency"), @@ -33,30 +39,37 @@ public enum SearchNodeMetrics implements VespaMetrics { CONTENT_PROTON_EXECUTOR_PROTON_ACCEPTED("content.proton.executor.proton.accepted", Unit.TASK, "Number of executor proton accepted tasks"), CONTENT_PROTON_EXECUTOR_PROTON_WAKEUPS("content.proton.executor.proton.wakeups", Unit.WAKEUP, "Number of times a executor proton worker thread has been woken up"), CONTENT_PROTON_EXECUTOR_PROTON_UTILIZATION("content.proton.executor.proton.utilization", Unit.FRACTION, "Ratio of time the executor proton worker threads has been active"), + CONTENT_PROTON_EXECUTOR_PROTON_REJECTED("content.proton.executor.proton.rejected", Unit.TASK, "Number of rejected tasks"), CONTENT_PROTON_EXECUTOR_FLUSH_QUEUESIZE("content.proton.executor.flush.queuesize", Unit.TASK, "Size of executor flush task queue"), CONTENT_PROTON_EXECUTOR_FLUSH_ACCEPTED("content.proton.executor.flush.accepted", Unit.TASK, "Number of accepted executor flush tasks"), CONTENT_PROTON_EXECUTOR_FLUSH_WAKEUPS("content.proton.executor.flush.wakeups", Unit.WAKEUP, "Number of times a executor flush worker thread has been woken up"), CONTENT_PROTON_EXECUTOR_FLUSH_UTILIZATION("content.proton.executor.flush.utilization", Unit.FRACTION, "Ratio of time the executor flush worker threads has been active"), + CONTENT_PROTON_EXECUTOR_FLUSH_REJECTED("content.proton.executor.flush.rejected", Unit.TASK, "Number of rejected tasks"), CONTENT_PROTON_EXECUTOR_MATCH_QUEUESIZE("content.proton.executor.match.queuesize", Unit.TASK, "Size of executor match task queue"), CONTENT_PROTON_EXECUTOR_MATCH_ACCEPTED("content.proton.executor.match.accepted", Unit.TASK, "Number of accepted executor match tasks"), CONTENT_PROTON_EXECUTOR_MATCH_WAKEUPS("content.proton.executor.match.wakeups", Unit.WAKEUP, "Number of times a executor match worker thread has been woken up"), CONTENT_PROTON_EXECUTOR_MATCH_UTILIZATION("content.proton.executor.match.utilization", Unit.FRACTION, "Ratio of time the executor match worker threads has been active"), + CONTENT_PROTON_EXECUTOR_MATCH_REJECTED("content.proton.executor.match.rejected", Unit.TASK, "Number of rejected tasks"), CONTENT_PROTON_EXECUTOR_DOCSUM_QUEUESIZE("content.proton.executor.docsum.queuesize", Unit.TASK, "Size of executor docsum task queue"), CONTENT_PROTON_EXECUTOR_DOCSUM_ACCEPTED("content.proton.executor.docsum.accepted", Unit.TASK, "Number of executor accepted docsum tasks"), CONTENT_PROTON_EXECUTOR_DOCSUM_WAKEUPS("content.proton.executor.docsum.wakeups", Unit.WAKEUP, "Number of times a executor docsum worker thread has been woken up"), CONTENT_PROTON_EXECUTOR_DOCSUM_UTILIZATION("content.proton.executor.docsum.utilization", Unit.FRACTION, "Ratio of time the executor docsum worker threads has been active"), + CONTENT_PROTON_EXECUTOR_DOCSUM_REJECTED("content.proton.executor.docsum.rejected", Unit.TASK, "Number of rejected tasks"), CONTENT_PROTON_EXECUTOR_SHARED_QUEUESIZE("content.proton.executor.shared.queuesize", Unit.TASK, "Size of executor shared task queue"), CONTENT_PROTON_EXECUTOR_SHARED_ACCEPTED("content.proton.executor.shared.accepted", Unit.TASK, "Number of executor shared accepted tasks"), CONTENT_PROTON_EXECUTOR_SHARED_WAKEUPS("content.proton.executor.shared.wakeups", Unit.WAKEUP, "Number of times a executor shared worker thread has been woken up"), CONTENT_PROTON_EXECUTOR_SHARED_UTILIZATION("content.proton.executor.shared.utilization", Unit.FRACTION, "Ratio of time the executor shared worker threads has been active"), + CONTENT_PROTON_EXECUTOR_SHARED_REJECTED("content.proton.executor.shared.rejected", Unit.TASK, "Number of rejected tasks"), CONTENT_PROTON_EXECUTOR_WARMUP_QUEUESIZE("content.proton.executor.warmup.queuesize", Unit.TASK, "Size of executor warmup task queue"), CONTENT_PROTON_EXECUTOR_WARMUP_ACCEPTED("content.proton.executor.warmup.accepted", Unit.TASK, "Number of accepted executor warmup tasks"), CONTENT_PROTON_EXECUTOR_WARMUP_WAKEUPS("content.proton.executor.warmup.wakeups", Unit.WAKEUP, "Number of times a warmup executor worker thread has been woken up"), CONTENT_PROTON_EXECUTOR_WARMUP_UTILIZATION("content.proton.executor.warmup.utilization", Unit.FRACTION, "Ratio of time the executor warmup worker threads has been active"), + CONTENT_PROTON_EXECUTOR_WARMUP_REJECTED("content.proton.executor.warmup.rejected", Unit.TASK, "Number of rejected tasks"), CONTENT_PROTON_EXECUTOR_FIELD_WRITER_QUEUESIZE("content.proton.executor.field_writer.queuesize", Unit.TASK, "Size of executor field writer task queue"), CONTENT_PROTON_EXECUTOR_FIELD_WRITER_ACCEPTED("content.proton.executor.field_writer.accepted", Unit.TASK, "Number of accepted executor field writer tasks"), CONTENT_PROTON_EXECUTOR_FIELD_WRITER_WAKEUPS("content.proton.executor.field_writer.wakeups", Unit.WAKEUP, "Number of times a executor field writer worker thread has been woken up"), CONTENT_PROTON_EXECUTOR_FIELD_WRITER_UTILIZATION("content.proton.executor.field_writer.utilization", Unit.FRACTION, "Ratio of time the executor fieldwriter worker threads has been active"), + CONTENT_PROTON_EXECUTOR_FIELD_WRITER_REJECTED("content.proton.executor.field_writer.rejected", Unit.TASK, "Number of rejected tasks"), // jobs CONTENT_PROTON_DOCUMENTDB_JOB_TOTAL("content.proton.documentdb.job.total", Unit.FRACTION, "The job load average total of all job metrics"), @@ -74,14 +87,32 @@ public enum SearchNodeMetrics implements VespaMetrics { CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_MASTER_ACCEPTED("content.proton.documentdb.threading_service.master.accepted", Unit.TASK, "Number of accepted threading service master tasks"), CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_MASTER_WAKEUPS("content.proton.documentdb.threading_service.master.wakeups", Unit.WAKEUP, "Number of times a threading service master worker thread has been woken up"), CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_MASTER_UTILIZATION("content.proton.documentdb.threading_service.master.utilization", Unit.FRACTION, "Ratio of time the threading service master worker threads has been active"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_MASTER_REJECTED("content.proton.documentdb.threading_service.master.rejected", Unit.TASK, "Number of rejected tasks"), CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_QUEUESIZE("content.proton.documentdb.threading_service.index.queuesize", Unit.TASK, "Size of threading service index task queue"), CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_ACCEPTED("content.proton.documentdb.threading_service.index.accepted", Unit.TASK, "Number of accepted threading service index tasks"), CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_WAKEUPS("content.proton.documentdb.threading_service.index.wakeups", Unit.WAKEUP, "Number of times a threading service index worker thread has been woken up"), CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_UTILIZATION("content.proton.documentdb.threading_service.index.utilization", Unit.FRACTION, "Ratio of time the threading service index worker threads has been active"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_REJECTED("content.proton.documentdb.threading_service.index.rejected", Unit.TASK, "Number of rejected tasks"), CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_QUEUESIZE("content.proton.documentdb.threading_service.summary.queuesize", Unit.TASK, "Size of threading service summary task queue"), CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_ACCEPTED("content.proton.documentdb.threading_service.summary.accepted", Unit.TASK, "Number of accepted threading service summary tasks"), CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_WAKEUPS("content.proton.documentdb.threading_service.summary.wakeups", Unit.WAKEUP, "Number of times a threading service summary worker thread has been woken up"), CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_UTILIZATION("content.proton.documentdb.threading_service.summary.utilization", Unit.FRACTION, "Ratio of time the threading service summary worker threads has been active"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_REJECTED("content.proton.documentdb.threading_service.summary.rejected", Unit.TASK, "Number of rejected tasks"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_ATTRIBUTE_FIELD_WRITER_ACCEPTED("content.proton.documentdb.threading_service.attribute_field_writer.accepted", Unit.TASK, "Number of accepted tasks"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_ATTRIBUTE_FIELD_WRITER_QUEUESIZE("content.proton.documentdb.threading_service.attribute_field_writer.queuesize", Unit.TASK, "Size of task queue"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_ATTRIBUTE_FIELD_WRITER_REJECTED("content.proton.documentdb.threading_service.attribute_field_writer.rejected", Unit.TASK, "Number of rejected tasks"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_ATTRIBUTE_FIELD_WRITER_UTILIZATION("content.proton.documentdb.threading_service.attribute_field_writer.utilization", Unit.FRACTION, "Ratio of time the worker threads has been active"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_ATTRIBUTE_FIELD_WRITER_WAKEUPS("content.proton.documentdb.threading_service.attribute_field_writer.wakeups", Unit.WAKEUP, "Number of times a worker thread has been woken up"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_INVERTER_ACCEPTED("content.proton.documentdb.threading_service.index_field_inverter.accepted", Unit.TASK, "Number of accepted tasks"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_INVERTER_QUEUESIZE("content.proton.documentdb.threading_service.index_field_inverter.queuesize", Unit.TASK, "Size of task queue"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_INVERTER_REJECTED("content.proton.documentdb.threading_service.index_field_inverter.rejected", Unit.TASK, "Number of rejected tasks"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_INVERTER_UTILIZATION("content.proton.documentdb.threading_service.index_field_inverter.utilization", Unit.FRACTION, "Ratio of time the worker threads has been active"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_INVERTER_WAKEUPS("content.proton.documentdb.threading_service.index_field_inverter.wakeups", Unit.WAKEUP, "Number of times a worker thread has been woken up"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_WRITER_ACCEPTED("content.proton.documentdb.threading_service.index_field_writer.accepted", Unit.TASK, "Number of accepted tasks"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_WRITER_QUEUESIZE("content.proton.documentdb.threading_service.index_field_writer.queuesize", Unit.TASK, "Size of task queue"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_WRITER_REJECTED("content.proton.documentdb.threading_service.index_field_writer.rejected", Unit.TASK, "Number of rejected tasks"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_WRITER_UTILIZATION("content.proton.documentdb.threading_service.index_field_writer.utilization", Unit.FRACTION, "Ratio of time the worker threads has been active"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_WRITER_WAKEUPS("content.proton.documentdb.threading_service.index_field_writer.wakeups", Unit.WAKEUP, "Number of times a worker thread has been woken up"), // lid space CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LID_BLOAT_FACTOR("content.proton.documentdb.ready.lid_space.lid_bloat_factor", Unit.FRACTION, "The bloat factor of this lid space, indicating the total amount of holes in the allocated lid space ((lid_limit - used_lids) / lid_limit)"), @@ -89,16 +120,19 @@ public enum SearchNodeMetrics implements VespaMetrics { CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LID_LIMIT("content.proton.documentdb.ready.lid_space.lid_limit", Unit.DOCUMENTID, "The size of the allocated lid space"), CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_HIGHEST_USED_LID("content.proton.documentdb.ready.lid_space.highest_used_lid", Unit.DOCUMENTID, "The highest used lid"), CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_USED_LIDS("content.proton.documentdb.ready.lid_space.used_lids", Unit.DOCUMENTID, "The number of lids used"), + CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LOWEST_FREE_LID("content.proton.documentdb.ready.lid_space.lowest_free_lid", Unit.DOCUMENTID, "The lowest free local document id"), CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_LID_BLOAT_FACTOR("content.proton.documentdb.notready.lid_space.lid_bloat_factor", Unit.FRACTION, "The bloat factor of this lid space, indicating the total amount of holes in the allocated lid space ((lid_limit - used_lids) / lid_limit)"), CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_LID_FRAGMENTATION_FACTOR("content.proton.documentdb.notready.lid_space.lid_fragmentation_factor", Unit.FRACTION, "The fragmentation factor of this lid space, indicating the amount of holes in the currently used part of the lid space ((highest_used_lid - used_lids) / highest_used_lid)"), CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_LID_LIMIT("content.proton.documentdb.notready.lid_space.lid_limit", Unit.DOCUMENTID, "The size of the allocated lid space"), CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_HIGHEST_USED_LID("content.proton.documentdb.notready.lid_space.highest_used_lid", Unit.DOCUMENTID, "The highest used lid"), CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_USED_LIDS("content.proton.documentdb.notready.lid_space.used_lids", Unit.DOCUMENTID, "The number of lids used"), + CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_LOWEST_FREE_LID("content.proton.documentdb.notready.lid_space.lowest_free_lid", Unit.DOCUMENTID, "The lowest free local document id"), CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_LID_BLOAT_FACTOR("content.proton.documentdb.removed.lid_space.lid_bloat_factor", Unit.FRACTION, "The bloat factor of this lid space, indicating the total amount of holes in the allocated lid space ((lid_limit - used_lids) / lid_limit)"), CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_LID_FRAGMENTATION_FACTOR("content.proton.documentdb.removed.lid_space.lid_fragmentation_factor", Unit.FRACTION, "The fragmentation factor of this lid space, indicating the amount of holes in the currently used part of the lid space ((highest_used_lid - used_lids) / highest_used_lid)"), CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_LID_LIMIT("content.proton.documentdb.removed.lid_space.lid_limit", Unit.DOCUMENTID, "The size of the allocated lid space"), CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_HIGHEST_USED_LID("content.proton.documentdb.removed.lid_space.highest_used_lid", Unit.DOCUMENTID, "The highest used lid"), CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_USED_LIDS("content.proton.documentdb.removed.lid_space.used_lids", Unit.DOCUMENTID, "The number of lids used"), + CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_LOWEST_FREE_LID("content.proton.documentdb.removed.lid_space.lowest_free_lid", Unit.DOCUMENTID, "The lowest free local document id"), // bucket move CONTENT_PROTON_DOCUMENTDB_BUCKET_MOVE_BUCKETS_PENDING("content.proton.documentdb.bucket_move.buckets_pending", Unit.BUCKET, "The number of buckets left to move"), @@ -118,6 +152,10 @@ public enum SearchNodeMetrics implements VespaMetrics { CONTENT_PROTON_RESOURCE_USAGE_MALLOC_ARENA("content.proton.resource_usage.malloc_arena", Unit.BYTE, "Size of malloc arena"), CONTENT_PROTON_DOCUMENTDB_ATTRIBUTE_RESOURCE_USAGE_ADDRESS_SPACE("content.proton.documentdb.attribute.resource_usage.address_space", Unit.FRACTION, "The max relative address space used among components in all attribute vectors in this document db (value in the range [0, 1])"), CONTENT_PROTON_DOCUMENTDB_ATTRIBUTE_RESOURCE_USAGE_FEEDING_BLOCKED("content.proton.documentdb.attribute.resource_usage.feeding_blocked", Unit.BINARY, "Whether feeding is blocked due to attribute resource limits being reached (value is either 0 or 1)"), + CONTENT_PROTON_DOCUMENTDB_ATTRIBUTE_MEMORY_USAGE_ALLOCATED_BYTES("content.proton.documentdb.attribute.memory_usage.allocated_bytes", Unit.BYTE, "The number of allocated bytes"), + CONTENT_PROTON_DOCUMENTDB_ATTRIBUTE_MEMORY_USAGE_DEAD_BYTES("content.proton.documentdb.attribute.memory_usage.dead_bytes", Unit.BYTE, "The number of dead bytes (<= used_bytes)"), + CONTENT_PROTON_DOCUMENTDB_ATTRIBUTE_MEMORY_USAGE_ONHOLD_BYTES("content.proton.documentdb.attribute.memory_usage.onhold_bytes", Unit.BYTE, "The number of bytes on hold"), + CONTENT_PROTON_DOCUMENTDB_ATTRIBUTE_MEMORY_USAGE_USED_BYTES("content.proton.documentdb.attribute.memory_usage.used_bytes", Unit.BYTE, "The number of used bytes (<= allocated_bytes)"), // CPU util CONTENT_PROTON_RESOURCE_USAGE_CPU_UTIL_SETUP("content.proton.resource_usage.cpu_util.setup", Unit.FRACTION, "cpu used by system init and (re-)configuration"), @@ -155,14 +193,21 @@ public enum SearchNodeMetrics implements VespaMetrics { CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_MEMORY_USAGE_ONHOLD_BYTES("content.proton.documentdb.removed.document_store.memory_usage.onhold_bytes", Unit.BYTE, "The number of bytes on hold"), // document store cache + CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_CACHE_ELEMENTS("content.proton.documentdb.ready.document_store.cache.elements", Unit.ITEM, "Number of elements in the cache"), CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_CACHE_MEMORY_USAGE("content.proton.documentdb.ready.document_store.cache.memory_usage", Unit.BYTE, "Memory usage of the cache (in bytes)"), CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_CACHE_HIT_RATE("content.proton.documentdb.ready.document_store.cache.hit_rate", Unit.FRACTION, "Rate of hits in the cache compared to number of lookups"), CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_CACHE_LOOKUPS("content.proton.documentdb.ready.document_store.cache.lookups", Unit.OPERATION, "Number of lookups in the cache (hits + misses)"), CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_CACHE_INVALIDATIONS("content.proton.documentdb.ready.document_store.cache.invalidations", Unit.OPERATION, "Number of invalidations (erased elements) in the cache. "), + CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_CACHE_ELEMENTS("content.proton.documentdb.notready.document_store.cache.elements", Unit.ITEM, "Number of elements in the cache"), CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_CACHE_MEMORY_USAGE("content.proton.documentdb.notready.document_store.cache.memory_usage", Unit.BYTE, "Memory usage of the cache (in bytes)"), CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_CACHE_HIT_RATE("content.proton.documentdb.notready.document_store.cache.hit_rate", Unit.FRACTION, "Rate of hits in the cache compared to number of lookups"), CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_CACHE_LOOKUPS("content.proton.documentdb.notready.document_store.cache.lookups", Unit.OPERATION, "Number of lookups in the cache (hits + misses)"), CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_CACHE_INVALIDATIONS("content.proton.documentdb.notready.document_store.cache.invalidations", Unit.OPERATION, "Number of invalidations (erased elements) in the cache. "), + CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_CACHE_ELEMENTS("content.proton.documentdb.removed.document_store.cache.elements", Unit.ITEM, "Number of elements in the cache"), + CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_CACHE_HIT_RATE("content.proton.documentdb.removed.document_store.cache.hit_rate", Unit.FRACTION, "Rate of hits in the cache compared to number of lookups"), + CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_CACHE_INVALIDATIONS("content.proton.documentdb.removed.document_store.cache.invalidations", Unit.ITEM, "Number of invalidations (erased elements) in the cache. "), + CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_CACHE_LOOKUPS("content.proton.documentdb.removed.document_store.cache.lookups", Unit.OPERATION, "Number of lookups in the cache (hits + misses)"), + CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_CACHE_MEMORY_USAGE("content.proton.documentdb.removed.document_store.cache.memory_usage", Unit.BYTE, "Memory usage of the cache (in bytes)"), // attribute CONTENT_PROTON_DOCUMENTDB_READY_ATTRIBUTE_MEMORY_USAGE_ALLOCATED_BYTES("content.proton.documentdb.ready.attribute.memory_usage.allocated_bytes", Unit.BYTE, "The number of allocated bytes"), @@ -179,6 +224,7 @@ public enum SearchNodeMetrics implements VespaMetrics { CONTENT_PROTON_DOCUMENTDB_INDEX_MEMORY_USAGE_USED_BYTES("content.proton.documentdb.index.memory_usage.used_bytes", Unit.BYTE, "The number of used bytes (<= allocated_bytes)"), CONTENT_PROTON_DOCUMENTDB_INDEX_MEMORY_USAGE_DEAD_BYTES("content.proton.documentdb.index.memory_usage.dead_bytes", Unit.BYTE, "The number of dead bytes (<= used_bytes)"), CONTENT_PROTON_DOCUMENTDB_INDEX_MEMORY_USAGE_ONHOLD_BYTES("content.proton.documentdb.index.memory_usage.onhold_bytes", Unit.BYTE, "The number of bytes on hold"), + CONTENT_PROTON_DOCUMENTDB_INDEX_DISK_USAGE("content.proton.documentdb.index.disk_usage", Unit.BYTE, "Disk space usage in bytes"), // matching CONTENT_PROTON_DOCUMENTDB_MATCHING_QUERIES("content.proton.documentdb.matching.queries", Unit.QUERY, "Number of queries executed"), @@ -186,6 +232,7 @@ public enum SearchNodeMetrics implements VespaMetrics { CONTENT_PROTON_DOCUMENTDB_MATCHING_QUERY_LATENCY("content.proton.documentdb.matching.query_latency", Unit.SECOND, "Total average latency (sec) when matching and ranking a query"), CONTENT_PROTON_DOCUMENTDB_MATCHING_QUERY_SETUP_TIME("content.proton.documentdb.matching.query_setup_time", Unit.SECOND, "Average time (sec) spent setting up and tearing down queries"), CONTENT_PROTON_DOCUMENTDB_MATCHING_DOCS_MATCHED("content.proton.documentdb.matching.docs_matched", Unit.DOCUMENT, "Number of documents matched"), + CONTENT_PROTON_DOCUMENTDB_MATCHING_DOCS_RANKED("content.proton.documentdb.matching.docs_ranked", Unit.DOCUMENT, "Number of documents ranked (first phase)"), CONTENT_PROTON_DOCUMENTDB_MATCHING_DOCS_RERANKED("content.proton.documentdb.matching.docs_reranked", Unit.DOCUMENT, "Number of documents re-ranked (second phase)"), CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_QUERIES("content.proton.documentdb.matching.rank_profile.queries", Unit.QUERY, "Number of queries executed"), CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_SOFT_DOOMED_QUERIES("content.proton.documentdb.matching.rank_profile.soft_doomed_queries", Unit.QUERY, "Number of queries hitting the soft timeout"), @@ -195,11 +242,39 @@ public enum SearchNodeMetrics implements VespaMetrics { CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_GROUPING_TIME("content.proton.documentdb.matching.rank_profile.grouping_time", Unit.SECOND, "Average time (sec) spent on grouping"), CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_RERANK_TIME("content.proton.documentdb.matching.rank_profile.rerank_time", Unit.SECOND, "Average time (sec) spent on 2nd phase ranking"), CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_DOCS_MATCHED("content.proton.documentdb.matching.rank_profile.docs_matched", Unit.DOCUMENT, "Number of documents matched"), + CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_DOCS_RANKED("content.proton.documentdb.matching.rank_profile.docs_ranked", Unit.DOCUMENT, "Number of documents ranked (first phase)"), + CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_DOCS_RERANKED("content.proton.documentdb.matching.rank_profile.docs_reranked", Unit.DOCUMENT, "Number of documents re-ranked (second phase)"), CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_LIMITED_QUERIES("content.proton.documentdb.matching.rank_profile.limited_queries", Unit.QUERY, "Number of queries limited in match phase"), + CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_DOCID_PARTITION_ACTIVE_TIME("content.proton.documentdb.matching.rank_profile.docid_partition.active_time", Unit.SECOND, "Time (sec) spent doing actual work"), + CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_DOCID_PARTITION_DOCS_MATCHED("content.proton.documentdb.matching.rank_profile.docid_partition.docs_matched", Unit.DOCUMENT, "Number of documents matched"), + CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_DOCID_PARTITION_DOCS_RANKED("content.proton.documentdb.matching.rank_profile.docid_partition.docs_ranked", Unit.DOCUMENT, "Number of documents ranked (first phase)"), + CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_DOCID_PARTITION_DOCS_RERANKED("content.proton.documentdb.matching.rank_profile.docid_partition.docs_reranked", Unit.DOCUMENT, "Number of documents re-ranked (second phase)"), + CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_DOCID_PARTITION_WAIT_TIME("content.proton.documentdb.matching.rank_profile.docid_partition.wait_time", Unit.SECOND, "Time (sec) spent waiting for other external threads and resources"), + CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_MATCH_TIME("content.proton.documentdb.matching.rank_profile.match_time", Unit.SECOND, "Average time (sec) for matching a query (1st phase)"), // feeding CONTENT_PROTON_DOCUMENTDB_FEEDING_COMMIT_OPERATIONS("content.proton.documentdb.feeding.commit.operations", Unit.OPERATION, "Number of operations included in a commit"), - CONTENT_PROTON_DOCUMENTDB_FEEDING_COMMIT_LATENCY("content.proton.documentdb.feeding.commit.latency", Unit.SECOND, "Latency for commit in seconds"); + CONTENT_PROTON_DOCUMENTDB_FEEDING_COMMIT_LATENCY("content.proton.documentdb.feeding.commit.latency", Unit.SECOND, "Latency for commit in seconds"), + + + // Metrics emitters not used in any metrics sets + CONTENT_PROTON_SESSION_CACHE_GROUPING_NUM_CACHED("content.proton.session_cache.grouping.num_cached", Unit.SESSION, "Number of currently cached sessions"), + CONTENT_PROTON_SESSION_CACHE_GROUPING_NUM_DROPPED("content.proton.session_cache.grouping.num_dropped", Unit.SESSION, "Number of dropped cached sessions"), + CONTENT_PROTON_SESSION_CACHE_GROUPING_NUM_INSERT("content.proton.session_cache.grouping.num_insert", Unit.SESSION, "Number of inserted sessions"), + CONTENT_PROTON_SESSION_CACHE_GROUPING_NUM_PICK("content.proton.session_cache.grouping.num_pick", Unit.SESSION, "Number if picked sessions"), + CONTENT_PROTON_SESSION_CACHE_GROUPING_NUM_TIMEDOUT("content.proton.session_cache.grouping.num_timedout", Unit.SESSION, "Number of timed out sessions"), + CONTENT_PROTON_SESSION_CACHE_SEARCH_NUM_CACHED("content.proton.session_cache.search.num_cached", Unit.SESSION, "Number of currently cached sessions"), + CONTENT_PROTON_SESSION_CACHE_SEARCH_NUM_DROPPED("content.proton.session_cache.search.num_dropped", Unit.SESSION, "Number of dropped cached sessions"), + CONTENT_PROTON_SESSION_CACHE_SEARCH_NUM_INSERT("content.proton.session_cache.search.num_insert", Unit.SESSION, "Number of inserted sessions"), + CONTENT_PROTON_SESSION_CACHE_SEARCH_NUM_PICK("content.proton.session_cache.search.num_pick", Unit.SESSION, "Number if picked sessions"), + CONTENT_PROTON_SESSION_CACHE_SEARCH_NUM_TIMEDOUT("content.proton.session_cache.search.num_timedout", Unit.SESSION, "Number of timed out sessions"), + + METRICMANAGER_PERIODICHOOKLATENCY("metricmanager.periodichooklatency", Unit.MILLISECOND, "Time in ms used to update a single periodic hook"), + METRICMANAGER_RESETLATENCY("metricmanager.resetlatency", Unit.MILLISECOND, "Time in ms used to reset all metrics."), + METRICMANAGER_SLEEPTIME("metricmanager.sleeptime", Unit.MILLISECOND, "Time in ms worker thread is sleeping"), + METRICMANAGER_SNAPSHOTHOOKLATENCY("metricmanager.snapshothooklatency", Unit.MILLISECOND, "Time in ms used to update a single snapshot hook"), + METRICMANAGER_SNAPSHOTLATENCY("metricmanager.snapshotlatency", Unit.MILLISECOND, "Time in ms used to take a snapshot"); + private final String name; private final Unit unit; diff --git a/container-core/src/main/java/com/yahoo/metrics/SentinelMetrics.java b/container-core/src/main/java/com/yahoo/metrics/SentinelMetrics.java new file mode 100644 index 00000000000..7711b7e75f4 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/SentinelMetrics.java @@ -0,0 +1,36 @@ +package com.yahoo.metrics; + +/** + * @author yngve + */ +public enum SentinelMetrics implements VespaMetrics { + + SENTINEL_RESTARTS("sentinel.restarts", Unit.RESTART, "Number of service restarts done by the sentinel"), + SENTINEL_TOTAL_RESTARTS("sentinel.totalRestarts", Unit.RESTART, "Total number of service restarts done by the sentinel since the sentinel was started"), + SENTINEL_UPTIME("sentinel.uptime", Unit.SECOND, "Time the sentinel has been running"), + SENTINEL_RUNNING("sentinel.running", Unit.INSTANCE, "Number of services the sentinel has running currently"); + + + private final String name; + private final Unit unit; + private final String description; + + SentinelMetrics(String name, Unit unit, String description) { + this.name = name; + this.unit = unit; + this.description = description; + } + + public String baseName() { + return name; + } + + public Unit unit() { + return unit; + } + + public String description() { + return description; + } + +} diff --git a/container-core/src/main/java/com/yahoo/metrics/SlobrokMetrics.java b/container-core/src/main/java/com/yahoo/metrics/SlobrokMetrics.java new file mode 100644 index 00000000000..8c30bf8e414 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/SlobrokMetrics.java @@ -0,0 +1,37 @@ +package com.yahoo.metrics; + +/** + * @author yngve + */ +public enum SlobrokMetrics implements VespaMetrics { + + SLOBROK_HEARTBEATS_FAILED("slobrok.heartbeats.failed", Unit.REQUEST, "Number of heartbeat requests failed"), + SLOBROK_REQUESTS_REGISTER("slobrok.requests.register", Unit.REQUEST, "Number of register requests received"), + SLOBROK_REQUESTS_MIRROR("slobrok.requests.mirror", Unit.REQUEST, "Number of mirroring requests received"), + SLOBROK_REQUESTS_ADMIN("slobrok.requests.admin", Unit.REQUEST, "Number of administrative requests received"), + SLOBROK_MISSING_CONSENSUS("slobrok.missing.consensus", Unit.SECOND, "Number of seconds without full consensus with all other brokers"); + + + private final String name; + private final Unit unit; + private final String description; + + SlobrokMetrics(String name, Unit unit, String description) { + this.name = name; + this.unit = unit; + this.description = description; + } + + public String baseName() { + return name; + } + + public Unit unit() { + return unit; + } + + public String description() { + return description; + } + +} diff --git a/container-core/src/main/java/com/yahoo/metrics/StorageMetrics.java b/container-core/src/main/java/com/yahoo/metrics/StorageMetrics.java index 2a59e5a9d92..d67b67d04b7 100644 --- a/container-core/src/main/java/com/yahoo/metrics/StorageMetrics.java +++ b/container-core/src/main/java/com/yahoo/metrics/StorageMetrics.java @@ -75,7 +75,26 @@ public enum StorageMetrics implements VespaMetrics { VDS_MERGETHROTTLER_LOCALLYEXECUTEDMERGES_OK("vds.mergethrottler.locallyexecutedmerges.ok", Unit.INSTANCE, "The number of successful merges for 'locallyexecutedmerges'"), VDS_MERGETHROTTLER_MERGECHAINS_OK("vds.mergethrottler.mergechains.ok", Unit.INSTANCE, "The number of successful merges for 'mergechains'"), VDS_MERGETHROTTLER_MERGECHAINS_FAILURES_BUSY("vds.mergethrottler.mergechains.failures.busy", Unit.INSTANCE, "The number of merges that failed because the storage node was busy"), - VDS_MERGETHROTTLER_MERGECHAINS_FAILURES_TOTAL("vds.mergethrottler.mergechains.failures.total", Unit.INSTANCE, "Sum of all failures"); + VDS_MERGETHROTTLER_MERGECHAINS_FAILURES_TOTAL("vds.mergethrottler.mergechains.failures.total", Unit.INSTANCE, "Sum of all failures"), + + + // C++ TLS metrics - these come from both the distributor and storage + VDS_SERVER_NETWORK_TLS_HANDSHAKES_FAILED("vds.server.network.tls-handshakes-failed", Unit.OPERATION, "Number of client or server connection attempts that failed during TLS handshaking"), + VDS_SERVER_NETWORK_PEER_AUTHORIZATION_FAILURES("vds.server.network.peer-authorization-failures", Unit.FAILURE, "Number of TLS connection attempts failed due to bad or missing peer certificate credentials"), + VDS_SERVER_NETWORK_CLIENT_TLS_CONNECTIONS_ESTABLISHED("vds.server.network.client.tls-connections-established", Unit.CONNECTION, "Number of secure mTLS connections established"), + VDS_SERVER_NETWORK_SERVER_TLS_CONNECTIONS_ESTABLISHED("vds.server.network.server.tls-connections-established", Unit.CONNECTION, "Number of secure mTLS connections established"), + VDS_SERVER_NETWORK_CLIENT_INSECURE_CONNECTIONS_ESTABLISHED("vds.server.network.client.insecure-connections-established", Unit.CONNECTION, "Number of insecure (plaintext) connections established"), + VDS_SERVER_NETWORK_SERVER_INSECURE_CONNECTIONS_ESTABLISHED("vds.server.network.server.insecure-connections-established", Unit.CONNECTION, "Number of insecure (plaintext) connections established"), + VDS_SERVER_NETWORK_TLS_CONNECTIONS_BROKEN("vds.server.network.tls-connections-broken", Unit.CONNECTION, "Number of TLS connections broken due to failures during frame encoding or decoding"), + VDS_SERVER_NETWORK_FAILED_TLS_CONFIG_RELOADS("vds.server.network.failed-tls-config-reloads", Unit.FAILURE, "Number of times background reloading of TLS config has failed"), + + // C++ capability metrics + VDS_SERVER_NETWORK_RPC_CAPABILITY_CHECKS_FAILED("vds.server.network.rpc-capability-checks-failed", Unit.FAILURE, "Number of RPC operations that failed to due one or more missing capabilities"), + VDS_SERVER_NETWORK_STATUS_CAPABILITY_CHECKS_FAILED("vds.server.network.status-capability-checks-failed", Unit.FAILURE, "Number of status page operations that failed to due one or more missing capabilities"), + + // C++ Fnet metrics + VDS_SERVER_FNET_NUM_CONNECTIONS("vds.server.fnet.num-connections", Unit.CONNECTION, "Total number of connection objects"); + private final String name; private final Unit unit; diff --git a/container-core/src/main/java/com/yahoo/metrics/Unit.java b/container-core/src/main/java/com/yahoo/metrics/Unit.java index bb7718ddb4c..7411b5b0ca4 100644 --- a/container-core/src/main/java/com/yahoo/metrics/Unit.java +++ b/container-core/src/main/java/com/yahoo/metrics/Unit.java @@ -9,9 +9,11 @@ public enum Unit { BINARY(BaseUnit.BINARY), BUCKET(BaseUnit.BUCKET), BYTE(BaseUnit.BYTE), + BYTE_PER_SECOND(BaseUnit.BYTE, BaseUnit.SECOND), CONNECTION(BaseUnit.CONNECTION), DOCUMENT(BaseUnit.DOCUMENT), DOCUMENTID(BaseUnit.DOCUMENTID), + FAILURE(BaseUnit.FAILURE), FILE(BaseUnit.FILE), FRACTION(BaseUnit.FRACTION), HIT(BaseUnit.HIT), @@ -21,6 +23,7 @@ public enum Unit { MILLISECOND(BaseUnit.MILLISECOND), NANOSECOND(BaseUnit.NANOSECOND), NODE(BaseUnit.NODE), + PACKET(BaseUnit.PACKET), OPERATION(BaseUnit.OPERATION), OPERATION_PER_SECOND(BaseUnit.OPERATION, BaseUnit.SECOND), QUERY(BaseUnit.QUERY), @@ -28,8 +31,10 @@ public enum Unit { RECORD(BaseUnit.RECORD), REQUEST(BaseUnit.REQUEST), RESPONSE(BaseUnit.RESPONSE), + RESTART(BaseUnit.RESTART), SCORE(BaseUnit.SCORE), SECOND(BaseUnit.SECOND), + SESSION(BaseUnit.SESSION), TASK(BaseUnit.TASK), THREAD(BaseUnit.THREAD), VERSION(BaseUnit.VERSION), @@ -69,6 +74,7 @@ public enum Unit { CONNECTION("connection"), DOCUMENT("document"), DOCUMENTID("documentid"), + FAILURE("failure"), FILE("file"), FRACTION("fraction"), HIT("hit"), @@ -78,12 +84,15 @@ public enum Unit { NANOSECOND("nanosecond", "ns"), NODE("node"), OPERATION("operation"), + PACKET("packet"), QUERY("query"), RECORD("record"), REQUEST("request"), RESPONSE("response"), + RESTART("restart"), SCORE("score"), SECOND("second", "s"), + SESSION("session"), TASK("task"), THREAD("thread"), VERSION("version"), diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/MetricAggregator.java b/container-core/src/main/java/com/yahoo/metrics/simple/MetricAggregator.java index 5fd9afa3e4c..d41d2981be8 100644 --- a/container-core/src/main/java/com/yahoo/metrics/simple/MetricAggregator.java +++ b/container-core/src/main/java/com/yahoo/metrics/simple/MetricAggregator.java @@ -46,9 +46,7 @@ class MetricAggregator implements Runnable { private void createSnapshot(Bucket toDelete) { Bucket toPresent = new Bucket(); for (Bucket b : buffer) { - if (b == null) { - continue; - } + if (b == null) continue; toPresent.merge(b); } dimensions.updateDimensionPersistence(toDelete, toPresent); diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/MetricReceiver.java b/container-core/src/main/java/com/yahoo/metrics/simple/MetricReceiver.java index 409e3651091..021f3a170d8 100644 --- a/container-core/src/main/java/com/yahoo/metrics/simple/MetricReceiver.java +++ b/container-core/src/main/java/com/yahoo/metrics/simple/MetricReceiver.java @@ -30,138 +30,6 @@ public class MetricReceiver { private final Object histogramDefinitionsLock = new Object(); private volatile Map<String, MetricSettings> metricSettings; - private static final class NullCounter extends Counter { - - NullCounter() { - super(null, null, null); - } - - @Override - public void add() { - } - - @Override - public void add(long n) { - } - - @Override - public void add(Point p) { - } - - @Override - public void add(long n, Point p) { - } - - @Override - public PointBuilder builder() { - return super.builder(); - } - } - - private static final class NullGauge extends Gauge { - NullGauge() { - super(null, null, null); - } - - @Override - public void sample(double x) { - } - - @Override - public void sample(double x, Point p) { - } - - @Override - public PointBuilder builder() { - return super.builder(); - } - - } - - public static final class MockReceiver extends MetricReceiver { - - private final ThreadLocalDirectory<Bucket, Sample> collection; - - private MockReceiver(ThreadLocalDirectory<Bucket, Sample> collection) { - super(collection, null); - this.collection = collection; - } - - public MockReceiver() { - this(new ThreadLocalDirectory<>(new MetricUpdater())); - } - - /** Gathers all data since last snapshot */ - public Bucket getSnapshot() { - Bucket merged = new Bucket(); - for (Bucket b : collection.fetch()) { - merged.merge(b, true); - } - return merged; - } - - /** Utility method for testing */ - public Point point(String dim, String val) { - return pointBuilder().set(dim, val).build(); - } - - } - - private static final class NullReceiver extends MetricReceiver { - - NullReceiver() { - super(null, null); - } - - @Override - public void update(Sample s) { - } - - @Override - public Counter declareCounter(String name) { - return new NullCounter(); - } - - @Override - public Counter declareCounter(String name, Point boundDimensions) { - return new NullCounter(); - } - - @Override - public Gauge declareGauge(String name) { - return new NullGauge(); - } - - @Override - public Gauge declareGauge(String name, Point boundDimensions) { - return new NullGauge(); - } - - @Override - public Gauge declareGauge(String name, Optional<Point> boundDimensions, MetricSettings customSettings) { - return null; - } - - @Override - public PointBuilder pointBuilder() { - return null; - } - - @Override - public Bucket getSnapshot() { - return null; - } - - @Override - void addMetricDefinition(String metricName, MetricSettings definition) { - } - - @Override - MetricSettings getMetricDefinition(String metricName) { - return null; - } - } - public MetricReceiver(ThreadLocalDirectory<Bucket, Sample> metricsCollection, AtomicReference<Bucket> currentSnapshot) { this.metricsCollection = metricsCollection; this.currentSnapshot = currentSnapshot; @@ -297,4 +165,137 @@ public class MetricReceiver { MetricSettings getMetricDefinition(String metricName) { return metricSettings.get(metricName); } + + private static final class NullCounter extends Counter { + + NullCounter() { + super(null, null, null); + } + + @Override + public void add() { + } + + @Override + public void add(long n) { + } + + @Override + public void add(Point p) { + } + + @Override + public void add(long n, Point p) { + } + + @Override + public PointBuilder builder() { + return super.builder(); + } + } + + private static final class NullGauge extends Gauge { + NullGauge() { + super(null, null, null); + } + + @Override + public void sample(double x) { + } + + @Override + public void sample(double x, Point p) { + } + + @Override + public PointBuilder builder() { + return super.builder(); + } + + } + + public static final class MockReceiver extends MetricReceiver { + + private final ThreadLocalDirectory<Bucket, Sample> collection; + + private MockReceiver(ThreadLocalDirectory<Bucket, Sample> collection) { + super(collection, null); + this.collection = collection; + } + + public MockReceiver() { + this(new ThreadLocalDirectory<>(new MetricUpdater())); + } + + /** Gathers all data since last snapshot */ + public Bucket getSnapshot() { + Bucket merged = new Bucket(); + for (Bucket b : collection.fetch()) { + merged.merge(b, true); + } + return merged; + } + + /** Utility method for testing */ + public Point point(String dim, String val) { + return pointBuilder().set(dim, val).build(); + } + + } + + private static final class NullReceiver extends MetricReceiver { + + NullReceiver() { + super(null, null); + } + + @Override + public void update(Sample s) { + } + + @Override + public Counter declareCounter(String name) { + return new NullCounter(); + } + + @Override + public Counter declareCounter(String name, Point boundDimensions) { + return new NullCounter(); + } + + @Override + public Gauge declareGauge(String name) { + return new NullGauge(); + } + + @Override + public Gauge declareGauge(String name, Point boundDimensions) { + return new NullGauge(); + } + + @Override + public Gauge declareGauge(String name, Optional<Point> boundDimensions, MetricSettings customSettings) { + return null; + } + + @Override + public PointBuilder pointBuilder() { + return null; + } + + @Override + public Bucket getSnapshot() { + return null; + } + + @Override + void addMetricDefinition(String metricName, MetricSettings definition) { + } + + @Override + MetricSettings getMetricDefinition(String metricName) { + return null; + } + } + } diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/SimpleMetricConsumer.java b/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/SimpleMetricConsumer.java index 0286684c34c..7cb2f495b3e 100644 --- a/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/SimpleMetricConsumer.java +++ b/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/SimpleMetricConsumer.java @@ -50,4 +50,6 @@ public class SimpleMetricConsumer implements MetricConsumer { return new Point(properties); } + public MetricReceiver receiver() { return receiver; } + } diff --git a/container-core/src/main/resources/configdefinitions/container.qr-searchers.def b/container-core/src/main/resources/configdefinitions/container.qr-searchers.def index e89ee40d792..034e7fc442b 100644 --- a/container-core/src/main/resources/configdefinitions/container.qr-searchers.def +++ b/container-core/src/main/resources/configdefinitions/container.qr-searchers.def @@ -71,3 +71,6 @@ searchcluster[].indexingmode enum { REALTIME, STREAMING } default=REALTIME ## Storage cluster route to use for search cluster if indexingmode is streaming. searchcluster[].storagecluster.routespec string default="" + +## Enable global phase ranking +searchcluster[].globalphase bool default=false diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index d7600cac927..a95e0061992 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -243,8 +243,8 @@ <error-prone-annotations.version>2.18.0</error-prone-annotations.version> <guava.version>27.1-jre</guava.version> <guice.version>4.2.3</guice.version> - <jackson2.version>2.13.4</jackson2.version> - <jackson-databind.version>2.13.4.2</jackson-databind.version> + <jackson2.version>2.14.2</jackson2.version> + <jackson-databind.version>2.14.2</jackson-databind.version> <javax.inject.version>1</javax.inject.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <javax.ws.rs-api.version>2.0.1</javax.ws.rs-api.version> diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index b1eeffc24cc..1611aea6af3 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -62,6 +62,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.collections.CollectionUtil.first; +import static com.yahoo.metrics.ContainerMetrics.APPLICATION_GENERATION; /** * @author Tony Vaagenes @@ -73,7 +74,6 @@ public final class ConfiguredApplication implements Application { private final Set<ClientProvider> startedClients = createIdentityHashSet(); private final Set<ServerProvider> startedServers = createIdentityHashSet(); private final SubscriberFactory subscriberFactory; - private final Metric metric; private final ContainerActivator activator; private final String configId; private final OsgiFramework osgiFramework; @@ -140,13 +140,11 @@ public final class ConfiguredApplication implements Application { public ConfiguredApplication(ContainerActivator activator, OsgiFramework osgiFramework, com.yahoo.jdisc.Timer timer, - SubscriberFactory subscriberFactory, - Metric metric) { + SubscriberFactory subscriberFactory) { this.activator = activator; this.osgiFramework = osgiFramework; this.timerSingleton = timer; this.subscriberFactory = subscriberFactory; - this.metric = metric; this.configId = System.getProperty("config.id"); this.slobrokConfigSubscriber = (subscriberFactory instanceof CloudSubscriberFactory) ? new SlobrokConfigSubscriber(configId) @@ -301,10 +299,7 @@ public final class ConfiguredApplication implements Application { startAndStopServers(currentServers); startAndRemoveClients(Container.get().getClientProviderRegistry().allComponents()); - - log.info("Switching to the latest deployed set of configurations and components. " + - "Application config generation: " + configurer.generation()); - metric.set("application_generation", configurer.generation(), metric.createContext(Map.of())); + signalActivation(); } private void activateContainer(ContainerBuilder builder, Runnable onPreviousContainerTermination) { @@ -321,6 +316,13 @@ public final class ConfiguredApplication implements Application { } } + private void signalActivation() { + log.info("Switching to the latest deployed set of configurations and components. " + + "Application config generation: " + configurer.generation()); + var metric = configurer.getComponent(Metric.class); + metric.set(APPLICATION_GENERATION.baseName(), configurer.generation(), metric.createContext(Map.of())); + } + private ContainerBuilder createBuilderWithGuiceBindings() { ContainerBuilder builder = activator.newContainerBuilder(); setupGuiceBindings(builder.guiceModules()); @@ -341,10 +343,10 @@ public final class ConfiguredApplication implements Application { tryReportFailedComponentGraphConstructionMetric(configurer, e); log.log(Level.SEVERE, "Reconfiguration failed, your application package must be fixed, unless this is a " + - "JNI reload issue: " + Exceptions.toMessageString(e), e); + "JNI reload issue: " + Exceptions.toMessageString(e), e); } catch (Error e) { com.yahoo.protect.Process.logAndDie("java.lang.Error on reconfiguration: We are probably in " + - "a bad state and will terminate", e); + "a bad state and will terminate", e); } } log.fine("Reconfiguration loop exited"); diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/JrtMetrics.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/JrtMetrics.java index ca6b41962fe..24bb862cad5 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/JrtMetrics.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/JrtMetrics.java @@ -1,8 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.metric; +// import com.yahoo.jdisc.Container; import com.yahoo.jdisc.Metric; import com.yahoo.jrt.TransportMetrics; +import com.yahoo.metrics.ContainerMetrics; import static com.yahoo.jrt.TransportMetrics.Snapshot; @@ -24,12 +26,12 @@ class JrtMetrics { void emitMetrics() { Snapshot snapshot = transportMetrics.snapshot(); Snapshot changesSincePrevious = snapshot.changesSince(previousSnapshot); - increment("jrt.transport.tls-certificate-verification-failures", changesSincePrevious.tlsCertificateVerificationFailures()); - increment("jrt.transport.peer-authorization-failures", changesSincePrevious.peerAuthorizationFailures()); - increment("jrt.transport.server.tls-connections-established", changesSincePrevious.serverTlsConnectionsEstablished()); - increment("jrt.transport.client.tls-connections-established", changesSincePrevious.clientTlsConnectionsEstablished()); - increment("jrt.transport.server.unencrypted-connections-established", changesSincePrevious.serverUnencryptedConnectionsEstablished()); - increment("jrt.transport.client.unencrypted-connections-established", changesSincePrevious.clientUnencryptedConnectionsEstablished()); + increment(ContainerMetrics.JRT_TRANSPORT_TLS_CERTIFICATE_VERIFICATION_FAILURES.baseName(), changesSincePrevious.tlsCertificateVerificationFailures()); + increment(ContainerMetrics.JRT_TRANSPORT_PEER_AUTHORIZATION_FAILURES.baseName(), changesSincePrevious.peerAuthorizationFailures()); + increment(ContainerMetrics.JRT_TRANSPORT_SERVER_TLS_CONNECIONTS_ESTABLISHED.baseName(), changesSincePrevious.serverTlsConnectionsEstablished()); + increment(ContainerMetrics.JRT_TRANSPORT_CLIENT_TLS_CONNECTIONS_ESTABLISHED.baseName(), changesSincePrevious.clientTlsConnectionsEstablished()); + increment(ContainerMetrics.JRT_TRANSPORT_CLIENT_UNENCRYPTED_CONNECTIONS_ESTABLISHED.baseName(), changesSincePrevious.serverUnencryptedConnectionsEstablished()); + increment(ContainerMetrics.JRT_TRANSPORT_CLIENT_UNENCRYPTED_CONNECTIONS_ESTABLISHED.baseName(), changesSincePrevious.clientUnencryptedConnectionsEstablished()); previousSnapshot = snapshot; } diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 729aebf2fc2..08d8d54bc53 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -20,6 +20,7 @@ import com.yahoo.search.Searcher; import com.yahoo.search.config.ClusterConfig; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.query.ParameterParser; +import com.yahoo.search.ranking.GlobalPhaseRanker; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; @@ -64,6 +65,8 @@ public class ClusterSearcher extends Searcher { private final VespaBackEndSearcher server; private final Executor executor; + private final GlobalPhaseRanker globalPhaseHelper; + private final boolean enableGlobalPhase; @Inject public ClusterSearcher(ComponentId id, @@ -73,10 +76,12 @@ public class ClusterSearcher extends Searcher { DocumentdbInfoConfig documentDbConfig, SchemaInfo schemaInfo, ComponentRegistry<Dispatcher> dispatchers, + GlobalPhaseRanker globalPhaseHelper, VipStatus vipStatus, VespaDocumentAccess access) { super(id); this.executor = executor; + this.globalPhaseHelper = globalPhaseHelper; int searchClusterIndex = clusterConfig.clusterId(); searchClusterName = clusterConfig.clusterName(); QrSearchersConfig.Searchcluster searchClusterConfig = getSearchClusterConfigFromClusterName(qrsConfig, searchClusterName); @@ -101,6 +106,7 @@ public class ClusterSearcher extends Searcher { server = searchDispatch(searchClusterIndex, searchClusterName, uniqueServerId, docSumParams, documentDbConfig, schemaInfo, dispatchers); } + enableGlobalPhase = searchClusterConfig.globalphase(); } private static QrSearchersConfig.Searchcluster getSearchClusterConfigFromClusterName(QrSearchersConfig config, String name) { @@ -159,7 +165,10 @@ public class ClusterSearcher extends Searcher { maxQueryCacheTimeout = DEFAULT_MAX_QUERY_CACHE_TIMEOUT; server = searcher; this.executor = executor; + this.globalPhaseHelper = null; + this.enableGlobalPhase = false; } + /** Do not use, for internal testing purposes only. **/ ClusterSearcher(Set<String> schemas) { this(schemas, null, null); @@ -169,7 +178,7 @@ public class ClusterSearcher extends Searcher { public void fill(com.yahoo.search.Result result, String summaryClass, Execution execution) { Query query = result.getQuery(); - VespaBackEndSearcher searcher = server; + Searcher searcher = server; if (searcher != null) { if (query.getTimeLeft() > 0) { searcher.fill(result, summaryClass, execution); @@ -190,7 +199,7 @@ public class ClusterSearcher extends Searcher { public Result search(Query query, Execution execution) { validateQueryTimeout(query); validateQueryCache(query); - VespaBackEndSearcher searcher = server; + Searcher searcher = server; if (searcher == null) { return new Result(query, ErrorMessage.createNoBackendsInService("Could not search")); } @@ -228,8 +237,21 @@ public class ClusterSearcher extends Searcher { } else { String docType = schemas.iterator().next(); query.getModel().setRestrict(docType); - return searcher.search(query, execution); + return perSchemaSearch(searcher, query, execution); + } + } + + private Result perSchemaSearch(Searcher searcher, Query query, Execution execution) { + Set<String> restrict = query.getModel().getRestrict(); + if (restrict.size() != 1) { + throw new IllegalStateException("perSchemaSearch must always be called with 1 schema, got: " + restrict.size()); + } + String schema = restrict.iterator().next(); + Result result = searcher.search(query, execution); + if (globalPhaseHelper != null && enableGlobalPhase) { + globalPhaseHelper.process(query, result, schema); } + return result; } private static void processResult(Query query, FutureTask<Result> task, Result mergedResult) { @@ -248,12 +270,12 @@ public class ClusterSearcher extends Searcher { Set<String> schemas = resolveSchemas(query, execution.context().getIndexFacts()); List<Query> queries = createQueries(query, schemas); if (queries.size() == 1) { - return searcher.search(queries.get(0), execution); + return perSchemaSearch(searcher, queries.get(0), execution); } else { Result mergedResult = new Result(query); List<FutureTask<Result>> pending = new ArrayList<>(queries.size()); for (Query q : queries) { - FutureTask<Result> task = new FutureTask<>(() -> searcher.search(q, execution)); + FutureTask<Result> task = new FutureTask<>(() -> perSchemaSearch(searcher, q, execution)); try { executor.execute(task); pending.add(task); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index 494b16304ea..a0673fee9a5 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -40,8 +40,6 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { /** for vespa-internal use only; consider renaming the summary class */ public static final String SORTABLE_ATTRIBUTES_SUMMARY_CLASS = "attributeprefetch"; - private static final CompoundName TRACE_DISABLE = new CompoundName("trace.disable"); - private String serverId; /** The set of all document databases available in the backend handled by this searcher */ @@ -240,7 +238,7 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { } void traceQuery(String sourceName, String type, Query query, int offset, int hits, int level, Optional<String> quotedSummaryClass) { - if ((query.getTrace().getLevel()<level) || query.properties().getBoolean(TRACE_DISABLE)) return; + if ((query.getTrace().getLevel()<level) || !query.getTrace().getQuery()) return; StringBuilder s = new StringBuilder(); s.append(sourceName).append(" ").append(type).append(" to dispatch: ") @@ -309,12 +307,12 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { quotedSummaryClass.ifPresent((String summaryClass) -> s.append(" summary=").append(summaryClass)); query.trace(s.toString(), false, level); - if (query.getTrace().isTraceable(level + 1)) { + if (query.getTrace().isTraceable(level + 1) && query.getTrace().getQuery()) { query.trace("Current state of query tree: " + new TextualQueryRepresentation(query.getModel().getQueryTree().getRoot()), false, level+1); } - if (query.getTrace().isTraceable(level + 2)) { + if (query.getTrace().isTraceable(level + 2) && query.getTrace().getQuery()) { query.trace("YQL+ representation: " + query.yqlRepresentation(), level+2); } } diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java index 898e348db92..2d6e059342e 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java @@ -10,6 +10,8 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.yql.MinimalQueryInserter; +import com.yahoo.yolean.chain.After; /** * Recursively replaces all instances of OrItems with WeakAndItems if the query property weakand.replace is true. @@ -17,6 +19,7 @@ import com.yahoo.search.searchchain.Execution; * * @author karowan */ +@After(MinimalQueryInserter.EXTERNAL_YQL) public class WeakAndReplacementSearcher extends Searcher { static final CompoundName WEAKAND_REPLACE = new CompoundName("weakAnd.replace"); static final CompoundName WAND_HITS = new CompoundName("wand.hits"); diff --git a/container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java new file mode 100644 index 00000000000..d2edb776c92 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java @@ -0,0 +1,14 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import com.yahoo.tensor.Tensor; + +import java.util.Collection; + +interface Evaluator { + Collection<String> needInputs(); + + Evaluator bind(String name, Tensor value); + + double evaluateScore(); +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java new file mode 100644 index 00000000000..2c6ab9e9367 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java @@ -0,0 +1,83 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import com.yahoo.component.annotation.Inject; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.ranking.RankProfilesEvaluator.GlobalPhaseData; +import com.yahoo.tensor.Tensor; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; +import java.util.logging.Logger; + +public class GlobalPhaseRanker { + + private static final Logger logger = Logger.getLogger(GlobalPhaseRanker.class.getName()); + private final RankProfilesEvaluatorFactory factory; + + @Inject + public GlobalPhaseRanker(RankProfilesEvaluatorFactory factory) { + this.factory = factory; + logger.info("using factory: " + factory); + } + + public void process(Query query, Result result, String schema) { + String rankProfile = query.getRanking().getProfile(); + GlobalPhaseData data = factory.evaluatorForSchema(schema) + .flatMap(evaluator -> evaluator.getGlobalPhaseData(rankProfile)) + .orElse(null); + if (data == null) return; + var functionEvaluatorSource = data.functionEvaluatorSource(); + var prepared = findFromQuery(query, data.needInputs()); + Supplier<Evaluator> supplier = () -> { + var evaluator = functionEvaluatorSource.get(); + var simple = new SimpleEvaluator(evaluator); + for (var entry : prepared) { + simple.bind(entry.name(), entry.value()); + } + return simple; + }; + int rerankCount = data.rerankCount(); + if (rerankCount < 0) + rerankCount = 100; + ResultReranker.rerankHits(result, new HitRescorer(supplier), rerankCount); + } + + record NameAndValue(String name, Tensor value) { } + + /* do this only once per query: */ + List<NameAndValue> findFromQuery(Query query, List<String> needInputs) { + List<NameAndValue> result = new ArrayList<>(); + var ranking = query.getRanking(); + var rankFeatures = ranking.getFeatures(); + var rankProps = ranking.getProperties().asMap(); + for (String needed : needInputs) { + var optRef = com.yahoo.searchlib.rankingexpression.Reference.simple(needed); + if (optRef.isEmpty()) continue; + var ref = optRef.get(); + if (ref.name().equals("constant")) { + // XXX in theory, we should be able to avoid this + result.add(new NameAndValue(needed, null)); + continue; + } + if (ref.isSimple() && ref.name().equals("query")) { + String queryFeatureName = ref.simpleArgument().get(); + // searchers are recommended to place query features here: + var feature = rankFeatures.getTensor(queryFeatureName); + if (feature.isPresent()) { + result.add(new NameAndValue(needed, feature.get())); + } else { + // but other ways of setting query features end up in the properties: + var objList = rankProps.get(queryFeatureName); + if (objList != null && objList.size() == 1 && objList.get(0) instanceof Tensor t) { + result.add(new NameAndValue(needed, t)); + } + } + } + } + return result; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java b/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java new file mode 100644 index 00000000000..cce6b42d323 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import com.yahoo.search.result.FeatureData; +import com.yahoo.search.result.Hit; +import static com.yahoo.searchlib.rankingexpression.Reference.RANKING_EXPRESSION_WRAPPER; + +import java.util.function.Supplier; +import java.util.logging.Logger; + +class HitRescorer { + + private static final Logger logger = Logger.getLogger(HitRescorer.class.getName()); + + private final Supplier<Evaluator> evaluatorSource; + + public HitRescorer(Supplier<Evaluator> evaluatorSource) { + this.evaluatorSource = evaluatorSource; + } + + boolean rescoreHit(Hit hit) { + var features = hit.getField("matchfeatures"); + if (features instanceof FeatureData matchFeatures) { + var scorer = evaluatorSource.get(); + for (String argName : scorer.needInputs()) { + var asTensor = matchFeatures.getTensor(argName); + if (asTensor == null) { + asTensor = matchFeatures.getTensor(alternate(argName)); + } + if (asTensor != null) { + scorer.bind(argName, asTensor); + } else { + logger.warning("Missing match-feature for Evaluator argument: " + argName); + return false; + } + } + double newScore = scorer.evaluateScore(); + hit.setRelevance(newScore); + return true; + } else { + logger.warning("Hit without match-features: " + hit); + return false; + } + } + + private static final String RE_PREFIX = RANKING_EXPRESSION_WRAPPER + "("; + private static final String RE_SUFFIX = ")"; + private static final int RE_PRE_LEN = RE_PREFIX.length(); + private static final int RE_SUF_LEN = RE_SUFFIX.length(); + + static String alternate(String argName) { + if (argName.startsWith(RE_PREFIX) && argName.endsWith(RE_SUFFIX)) { + return argName.substring(RE_PRE_LEN, argName.length() - RE_SUF_LEN); + } + return argName; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java new file mode 100644 index 00000000000..2ca91a3ea91 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java @@ -0,0 +1,97 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.ranking; + +import ai.vespa.models.evaluation.FunctionEvaluator; +import ai.vespa.models.evaluation.Model; +import ai.vespa.models.evaluation.ModelsEvaluator; +import com.yahoo.api.annotations.Beta; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.annotation.Inject; +import com.yahoo.filedistribution.fileacquirer.FileAcquirer; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; +import java.util.logging.Logger; + +/** + * proxy for model-evaluation components + * @author arnej + */ +@Beta +public class RankProfilesEvaluator extends AbstractComponent { + + private final ModelsEvaluator evaluator; + private static final Logger logger = Logger.getLogger(RankProfilesEvaluator.class.getName()); + + @Inject + public RankProfilesEvaluator( + RankProfilesConfig rankProfilesConfig, + RankingConstantsConfig constantsConfig, + RankingExpressionsConfig expressionsConfig, + OnnxModelsConfig onnxModelsConfig, + FileAcquirer fileAcquirer) + { + this.evaluator = new ModelsEvaluator( + rankProfilesConfig, + constantsConfig, + expressionsConfig, + onnxModelsConfig, + fileAcquirer); + extractGlobalPhaseData(rankProfilesConfig); + } + + public Model modelForRankProfile(String rankProfile) { + var m = evaluator.models().get(rankProfile); + if (m == null) { + throw new IllegalArgumentException("unknown rankprofile: " + rankProfile); + } + return m; + } + + public FunctionEvaluator evaluatorForFunction(String rankProfile, String functionName) { + return modelForRankProfile(rankProfile).evaluatorOf(functionName); + } + + static record GlobalPhaseData(Supplier<FunctionEvaluator> functionEvaluatorSource, + int rerankCount, + List<String> needInputs) {} + + private Map<String, GlobalPhaseData> profilesWithGlobalPhase = new HashMap<>(); + + Optional<GlobalPhaseData> getGlobalPhaseData(String rankProfile) { + return Optional.ofNullable(profilesWithGlobalPhase.get(rankProfile)); + } + + private void extractGlobalPhaseData(RankProfilesConfig rankProfilesConfig) { + for (var rp : rankProfilesConfig.rankprofile()) { + String name = rp.name(); + Supplier<FunctionEvaluator> functionEvaluatorSource = null; + int rerankCount = -1; + List<String> needInputs = null; + + for (var prop : rp.fef().property()) { + if (prop.name().equals("vespa.globalphase.rerankcount")) { + rerankCount = Integer.valueOf(prop.value()); + } + if (prop.name().equals("vespa.rank.globalphase")) { + var model = modelForRankProfile(name); + functionEvaluatorSource = () -> model.evaluatorOf("globalphase"); + var evaluator = functionEvaluatorSource.get(); + needInputs = List.copyOf(evaluator.function().arguments()); + } + } + if (functionEvaluatorSource != null && needInputs != null) { + profilesWithGlobalPhase.put(name, new GlobalPhaseData(functionEvaluatorSource, rerankCount, needInputs)); + } + } + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluatorFactory.java b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluatorFactory.java new file mode 100644 index 00000000000..33f2fb74da5 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluatorFactory.java @@ -0,0 +1,39 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.ranking; + +import com.yahoo.api.annotations.Beta; +import com.yahoo.component.annotation.Inject; +import com.yahoo.component.provider.ComponentRegistry; + +import java.util.Optional; + +/** + * factory for model-evaluation proxies + * @author arnej + */ +@Beta +public class RankProfilesEvaluatorFactory { + + private final ComponentRegistry<RankProfilesEvaluator> registry; + + @Inject + public RankProfilesEvaluatorFactory(ComponentRegistry<RankProfilesEvaluator> registry) { + this.registry = registry; + } + + public Optional<RankProfilesEvaluator> evaluatorForSchema(String schemaName) { + return Optional.ofNullable(registry.getComponent("ranking-expression-evaluator." + schemaName)); + } + + @Override + public String toString() { + var buf = new StringBuilder(); + buf.append(this.getClass().getName()).append(" containing: ["); + for (var id : registry.allComponentsById().keySet()) { + buf.append(" ").append(id.toString()); + } + buf.append(" ]"); + return buf.toString(); + } +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java b/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java new file mode 100644 index 00000000000..8d24acdf141 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java @@ -0,0 +1,93 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.logging.Logger; + +class ResultReranker { + + private static final Logger logger = Logger.getLogger(ResultReranker.class.getName()); + + // scale and adjust the score according to the range + // of the original and final score values to avoid that + // a score from the backend is larger than finalScores_low + static class Ranges { + private double initialScores_high = -Double.MAX_VALUE; + private double initialScores_low = Double.MAX_VALUE; + private double finalScores_high = -Double.MAX_VALUE; + private double finalScores_low = Double.MAX_VALUE; + + boolean rescaleNeeded() { + return (initialScores_low > finalScores_low + && + initialScores_high >= initialScores_low + && + finalScores_high >= finalScores_low); + } + void withInitialScore(double score) { + if (score < initialScores_low) initialScores_low = score; + if (score > initialScores_high) initialScores_high = score; + } + void withFinalScore(double score) { + if (score < finalScores_low) finalScores_low = score; + if (score > finalScores_high) finalScores_high = score; + } + private double initialRange() { + double r = initialScores_high - initialScores_low; + if (r < 1.0) r = 1.0; + return r; + } + private double finalRange() { + double r = finalScores_high - finalScores_low; + if (r < 1.0) r = 1.0; + return r; + } + double scale() { return finalRange() / initialRange(); } + double bias() { return finalScores_low - initialScores_low * scale(); } + } + + static void rerankHits(Result result, HitRescorer hitRescorer, int rerankCount) { + List<Hit> hitsToRescore = new ArrayList<>(); + // consider doing recursive iteration explicitly instead of using deepIterator? + for (var iterator = result.hits().deepIterator(); iterator.hasNext();) { + Hit hit = iterator.next(); + if (hit.isMeta() || hit instanceof HitGroup) { + continue; + } + // what about hits inside grouping results? + // they are inside GroupingListHit, we won't recurse into it; so we won't see them. + hitsToRescore.add(hit); + } + // we can't be 100% certain that hits were sorted according to relevance: + hitsToRescore.sort(Comparator.naturalOrder()); + var ranges = new Ranges(); + for (var iterator = hitsToRescore.iterator(); rerankCount > 0 && iterator.hasNext(); ) { + Hit hit = iterator.next(); + double oldScore = hit.getRelevance().getScore(); + boolean didRerank = hitRescorer.rescoreHit(hit); + if (didRerank) { + ranges.withInitialScore(oldScore); + ranges.withFinalScore(hit.getRelevance().getScore()); + --rerankCount; + iterator.remove(); + } + } + // if any hits are left in the list, they may need rescaling: + if (ranges.rescaleNeeded()) { + double scale = ranges.scale(); + double bias = ranges.bias(); + for (Hit hit : hitsToRescore) { + double oldScore = hit.getRelevance().getScore(); + hit.setRelevance(oldScore * scale + bias); + } + } + result.hits().sort(); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java new file mode 100644 index 00000000000..f247eab1649 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import ai.vespa.models.evaluation.FunctionEvaluator; +import com.yahoo.search.result.FeatureData; +import com.yahoo.search.result.Hit; +import com.yahoo.tensor.Tensor; + +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +class SimpleEvaluator implements Evaluator { + + private final FunctionEvaluator evaluator; + private final Set<String> neededInputs; + + public SimpleEvaluator(FunctionEvaluator prototype) { + this.evaluator = prototype; + this.neededInputs = new HashSet<String>(prototype.function().arguments()); + } + + @Override + public Collection<String> needInputs() { return List.copyOf(neededInputs); } + + @Override + public SimpleEvaluator bind(String name, Tensor value) { + if (value != null) evaluator.bind(name, value); + neededInputs.remove(name); + return this; + } + + @Override + public double evaluateScore() { + return evaluator.evaluate().asDouble(); + } + + @Override + public String toString() { + var buf = new StringBuilder(); + buf.append("SimpleEvaluator("); + buf.append(evaluator.function().toString()); + buf.append(")["); + for (String arg : neededInputs) { + buf.append("{").append(arg).append("}"); + } + buf.append("]"); + return buf.toString(); + } +} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/package-info.java b/container-search/src/main/java/com/yahoo/search/ranking/package-info.java index 118f4b08c2a..a86a5c1e52f 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/package-info.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/package-info.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author mpolden - */ + @ExportPackage -package com.yahoo.vespa.hosted.ca.restapi; +package com.yahoo.search.ranking; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java index 421f19475a6..7e9fa3f748a 100644 --- a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java +++ b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java @@ -11,6 +11,7 @@ import com.yahoo.io.GrowableByteBuffer; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.serialization.JsonFormat; import com.yahoo.tensor.serialization.TypedBinaryFormat; +import static com.yahoo.searchlib.rankingexpression.Reference.wrapInRankingExpression; import java.nio.charset.StandardCharsets; import java.util.Collections; @@ -144,7 +145,7 @@ public class FeatureData implements Inspectable, JsonProducer { if (featureValue.valid()) return featureValue; // Try to wrap by rankingExpression(name) - return value.field("rankingExpression(" + featureName + ")"); + return value.field(wrapInRankingExpression(featureName)); } /** Returns the names of the features available in this */ diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index 5df8d2e5444..06ae9923dae 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -464,6 +464,7 @@ public class ClusterSearcherTestCase { documentDbConfig.build(), new SchemaInfo(List.of(schema.build()), Map.of()), dispatchers, + null, vipStatus, null); } diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 43aaba7b0f9..33f840c7af0 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -44,11 +44,9 @@ import com.yahoo.search.query.Sorting.Order; import com.yahoo.search.query.Sorting.UcaSorter; import com.yahoo.search.query.parser.Parsable; import com.yahoo.search.query.parser.ParserEnvironment; -import com.yahoo.search.query.parser.ParserFactory; import com.yahoo.search.searchchain.Execution; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; import java.util.ArrayList; import java.util.Collection; @@ -68,7 +66,6 @@ public class YqlParserTestCase { private final YqlParser parser = new YqlParser(new ParserEnvironment()); @Test - @Timeout(120_000) void failsGracefullyOnMissingQuoteEscapingAndSubsequentUnicodeCharacter() { assertParseFail("select * from bar where rank(ids contains 'http://en.wikipedia.org/wiki/Hors_d'Å“uvre') limit 10", new IllegalInputException("com.yahoo.search.yql.ProgramCompileException: query:L1:79 token recognition error at: 'Å“'")); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBuckets.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBuckets.java new file mode 100644 index 00000000000..62e341c674c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBuckets.java @@ -0,0 +1,34 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.archive; + +import java.util.HashSet; +import java.util.Set; + +/** + * @author freva + */ +public record ArchiveBuckets(Set<VespaManagedArchiveBucket> vespaManaged, + Set<TenantManagedArchiveBucket> tenantManaged) { + public static final ArchiveBuckets EMPTY = new ArchiveBuckets(Set.of(), Set.of()); + + public ArchiveBuckets(Set<VespaManagedArchiveBucket> vespaManaged, Set<TenantManagedArchiveBucket> tenantManaged) { + this.vespaManaged = Set.copyOf(vespaManaged); + this.tenantManaged = Set.copyOf(tenantManaged); + } + + /** Adds or replaces a VespaManagedArchive bucket with the given archive bucket */ + public ArchiveBuckets with(VespaManagedArchiveBucket vespaManagedArchiveBucket) { + Set<VespaManagedArchiveBucket> updated = new HashSet<>(vespaManaged); + updated.removeIf(bucket -> bucket.bucketName().equals(vespaManagedArchiveBucket.bucketName())); + updated.add(vespaManagedArchiveBucket); + return new ArchiveBuckets(updated, tenantManaged); + } + + /** Adds or replaces a TenantManagedArchive bucket with the given archive bucket */ + public ArchiveBuckets with(TenantManagedArchiveBucket tenantManagedArchiveBucket) { + Set<TenantManagedArchiveBucket> updated = new HashSet<>(tenantManaged); + updated.removeIf(bucket -> bucket.cloudAccount().equals(tenantManagedArchiveBucket.cloudAccount())); + updated.add(tenantManagedArchiveBucket); + return new ArchiveBuckets(vespaManaged, updated); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java index 46e7fb48553..ed965f4331e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java @@ -1,12 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.archive; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import java.net.URI; import java.util.Map; +import java.util.Optional; import java.util.Set; /** @@ -17,11 +19,13 @@ import java.util.Set; */ public interface ArchiveService { - ArchiveBucket createArchiveBucketFor(ZoneId zoneId); + VespaManagedArchiveBucket createArchiveBucketFor(ZoneId zoneId); - void updatePolicies(ZoneId zoneId, Set<ArchiveBucket> buckets, Map<TenantName,ArchiveAccess> authorizeAccessByTenantName); + void updatePolicies(ZoneId zoneId, Set<VespaManagedArchiveBucket> buckets, Map<TenantName,ArchiveAccess> authorizeAccessByTenantName); - boolean canAddTenantToBucket(ZoneId zoneId, ArchiveBucket bucket); + boolean canAddTenantToBucket(ZoneId zoneId, VespaManagedArchiveBucket bucket); - URI bucketURI(ZoneId zoneId, String bucketName, TenantName tenantName); + Optional<String> findEnclaveArchiveBucket(ZoneId zoneId, CloudAccount cloudAccount); + + URI bucketURI(ZoneId zoneId, String bucketName); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveUriUpdate.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveUriUpdate.java new file mode 100644 index 00000000000..e6dec99b84c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveUriUpdate.java @@ -0,0 +1,43 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.archive; + +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.TenantName; + +import java.net.URI; +import java.util.Optional; + +/** + * Represents an operation to update or unset the archive URI value for a given tenant or cloud account. + * + * @author freva + */ +public class ArchiveUriUpdate { + private final Optional<TenantName> tenantName; + private final Optional<CloudAccount> cloudAccount; + private final Optional<URI> archiveUri; + + private ArchiveUriUpdate(Optional<TenantName> tenantName, Optional<CloudAccount> cloudAccount, Optional<URI> archiveUri) { + this.tenantName = tenantName; + this.cloudAccount = cloudAccount; + this.archiveUri = archiveUri; + } + + public Optional<TenantName> tenantName() { return tenantName; } + public Optional<CloudAccount> cloudAccount() { return cloudAccount; } + public Optional<URI> archiveUri() { return archiveUri; } + + public static ArchiveUriUpdate setArchiveUriFor(TenantName tenantName, URI archiveUri) { + return new ArchiveUriUpdate(Optional.of(tenantName), Optional.empty(), Optional.of(archiveUri)); + } + public static ArchiveUriUpdate deleteArchiveUriFor(TenantName tenantName) { + return new ArchiveUriUpdate(Optional.of(tenantName), Optional.empty(), Optional.empty()); + } + + public static ArchiveUriUpdate setArchiveUriFor(CloudAccount cloudAccount, URI archiveUri) { + return new ArchiveUriUpdate(Optional.empty(), Optional.of(cloudAccount), Optional.of(archiveUri)); + } + public static ArchiveUriUpdate deleteArchiveUriFor(CloudAccount cloudAccount) { + return new ArchiveUriUpdate(Optional.empty(), Optional.of(cloudAccount), Optional.empty()); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java index a2847439ce7..7461d3aa47e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java @@ -1,16 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.archive; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import java.net.URI; +import java.time.Clock; import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Optional; import java.util.Set; -import java.util.TreeMap; /** * @author freva @@ -18,29 +20,54 @@ import java.util.TreeMap; */ public class MockArchiveService implements ArchiveService { - - public Set<ArchiveBucket> archiveBuckets = new HashSet<>(); + private final Map<ZoneId, Set<TenantManagedArchiveBucket>> tenantArchiveBucketsByZone = new HashMap<>(); + public Set<VespaManagedArchiveBucket> archiveBuckets = new HashSet<>(); public Map<TenantName, ArchiveAccess> authorizeAccessByTenantName = new HashMap<>(); + private final Clock clock; + + public MockArchiveService(Clock clock) { + this.clock = clock; + } @Override - public ArchiveBucket createArchiveBucketFor(ZoneId zoneId) { - return new ArchiveBucket("bucketName", "keyArn"); + public VespaManagedArchiveBucket createArchiveBucketFor(ZoneId zoneId) { + return new VespaManagedArchiveBucket("bucketName", "keyArn"); } @Override - public void updatePolicies(ZoneId zoneId, Set<ArchiveBucket> buckets, Map<TenantName, ArchiveAccess> authorizeAccessByTenantName) { + public void updatePolicies(ZoneId zoneId, Set<VespaManagedArchiveBucket> buckets, Map<TenantName, ArchiveAccess> authorizeAccessByTenantName) { this.archiveBuckets = new HashSet<>(buckets); this.authorizeAccessByTenantName = new HashMap<>(authorizeAccessByTenantName); } @Override - public boolean canAddTenantToBucket(ZoneId zoneId, ArchiveBucket bucket) { + public boolean canAddTenantToBucket(ZoneId zoneId, VespaManagedArchiveBucket bucket) { return bucket.tenants().size() < 5; } @Override - public URI bucketURI(ZoneId zoneId, String bucketName, TenantName tenantName) { - return URI.create(String.format("s3://%s/%s/", bucketName, tenantName.value())); + public Optional<String> findEnclaveArchiveBucket(ZoneId zoneId, CloudAccount cloudAccount) { + return tenantArchiveBucketsByZone.getOrDefault(zoneId, Set.of()).stream() + .filter(bucket -> bucket.cloudAccount().equals(cloudAccount)) + .findFirst() + .map(TenantManagedArchiveBucket::bucketName); + } + + @Override + public URI bucketURI(ZoneId zoneId, String bucketName) { + return URI.create(String.format("s3://%s/", bucketName)); + } + + + public void setEnclaveArchiveBucket(ZoneId zoneId, CloudAccount cloudAccount, String bucketName) { + removeEnclaveArchiveBucket(zoneId, cloudAccount); + tenantArchiveBucketsByZone.computeIfAbsent(zoneId, z -> new HashSet<>()) + .add(new TenantManagedArchiveBucket(bucketName, cloudAccount, clock.instant())); + } + + public void removeEnclaveArchiveBucket(ZoneId zoneId, CloudAccount cloudAccount) { + Optional.ofNullable(tenantArchiveBucketsByZone.get(zoneId)) + .ifPresent(set -> set.removeIf(bucket -> bucket.cloudAccount().equals(cloudAccount))); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/TenantManagedArchiveBucket.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/TenantManagedArchiveBucket.java new file mode 100644 index 00000000000..80e9762f84b --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/TenantManagedArchiveBucket.java @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.archive; + +import com.yahoo.config.provision.CloudAccount; + +import java.time.Instant; + +/** + * Represents a cloud storage bucket (e.g. AWS S3 or Google Storage) used to store archive data - logs, heap/core dumps, etc. + * that is managed by the tenant directly. + * + * @author freva + */ +public record TenantManagedArchiveBucket(String bucketName, CloudAccount cloudAccount, Instant updatedAt) { +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucket.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/VespaManagedArchiveBucket.java index be3b87ddc5c..c80e9b3780d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucket.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/VespaManagedArchiveBucket.java @@ -8,20 +8,21 @@ import java.util.Objects; import java.util.Set; /** - * Represents an S3 bucket used to store archive data - logs, heap/core dumps, etc. + * Represents a cloud storage bucket (e.g. AWS S3 or Google Storage) used to store archive data - logs, heap/core dumps, etc. + * that is managed by the Vespa controller. * * @author andreer */ -public class ArchiveBucket { +public class VespaManagedArchiveBucket { private final String bucketName; private final String keyArn; private final Set<TenantName> tenants; - public ArchiveBucket(String bucketName, String keyArn) { + public VespaManagedArchiveBucket(String bucketName, String keyArn) { this(bucketName, keyArn, Set.of()); } - private ArchiveBucket(String bucketName, String keyArn, Set<TenantName> tenants) { + private VespaManagedArchiveBucket(String bucketName, String keyArn, Set<TenantName> tenants) { this.bucketName = bucketName; this.keyArn = keyArn; this.tenants = Set.copyOf(tenants); @@ -39,19 +40,19 @@ public class ArchiveBucket { return tenants; } - public ArchiveBucket withTenant(TenantName tenant) { + public VespaManagedArchiveBucket withTenant(TenantName tenant) { return withTenants(Set.of(tenant)); } - public ArchiveBucket withTenants(Set<TenantName> tenants) { - return new ArchiveBucket(bucketName, keyArn, Sets.union(this.tenants, tenants)); + public VespaManagedArchiveBucket withTenants(Set<TenantName> tenants) { + return new VespaManagedArchiveBucket(bucketName, keyArn, Sets.union(this.tenants, tenants)); } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - ArchiveBucket that = (ArchiveBucket) o; + VespaManagedArchiveBucket that = (VespaManagedArchiveBucket) o; return bucketName.equals(that.bucketName) && keyArn.equals(that.keyArn) && tenants.equals(that.tenants); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java index 54fda58d19c..c4194315922 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java @@ -39,7 +39,7 @@ public class AthenzClientFactoryMock extends AbstractComponent implements Athenz @Override public ZtsClient createZtsClient() { - return new ZtsClientMock(athenz); + return new ZtsClientMock(athenz, createZmsClient()); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java index d3e74965c4b..3ca0fdd0f23 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java @@ -5,10 +5,12 @@ import com.yahoo.security.Pkcs10Csr; import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.AwsRole; import com.yahoo.vespa.athenz.api.AwsTemporaryCredentials; import com.yahoo.vespa.athenz.api.ZToken; +import com.yahoo.vespa.athenz.client.zms.ZmsClient; import com.yahoo.vespa.athenz.client.zts.Identity; import com.yahoo.vespa.athenz.client.zts.InstanceIdentity; import com.yahoo.vespa.athenz.client.zts.ZtsClient; @@ -17,6 +19,7 @@ import java.security.KeyPair; import java.security.cert.X509Certificate; import java.time.Duration; import java.util.List; +import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; @@ -27,9 +30,14 @@ public class ZtsClientMock implements ZtsClient { private static final Logger log = Logger.getLogger(ZtsClientMock.class.getName()); private final AthenzDbMock athenz; + private final Optional<ZmsClient> zmsClient; public ZtsClientMock(AthenzDbMock athenz) { + this(athenz, null); + } + public ZtsClientMock(AthenzDbMock athenz, ZmsClient zmsClient) { this.athenz = athenz; + this.zmsClient = Optional.ofNullable(zmsClient); } @Override @@ -98,6 +106,12 @@ public class ZtsClientMock implements ZtsClient { } @Override + public boolean hasAccess(AthenzResourceName resource, String action, AthenzIdentity identity) { + return zmsClient.orElseThrow(UnsupportedOperationException::new) + .hasAccess(resource, action, identity); + } + + @Override public void close() { } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java index 8b0f58c79d2..8b2f4187f65 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java @@ -92,7 +92,7 @@ public interface BillingController { void updateBillStatus(Bill.Id billId, String agent, String status); /** Add a line item to the given bill */ - void addLineItem(TenantName tenant, String description, BigDecimal amount, String agent); + void addLineItem(TenantName tenant, String description, BigDecimal amount, Optional<Bill.Id> billId, String agent); /** Delete a line item - only available for unused line items */ void deleteLineItem(String lineItemId); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java index 6ea4c7442d8..aa06e282e1c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java @@ -14,7 +14,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.stream.Collectors; /** * @author olaa @@ -122,16 +121,19 @@ public class MockBillingController implements BillingController { } @Override - public void addLineItem(TenantName tenant, String description, BigDecimal amount, String agent) { - unusedLineItems.computeIfAbsent(tenant, l -> new ArrayList<>()) - .add(new Bill.LineItem( - "line-item-id", - description, - amount, - "some-plan", - agent, - ZonedDateTime.now() - )); + public void addLineItem(TenantName tenant, String description, BigDecimal amount, Optional<Bill.Id> billId, String agent) { + if (billId.isPresent()) { + throw new UnsupportedOperationException(); + } else { + unusedLineItems.computeIfAbsent(tenant, l -> new ArrayList<>()) + .add(new Bill.LineItem( + "line-item-id", + description, + amount, + "some-plan", + agent, + ZonedDateTime.now())); + } } @Override diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ArchiveUris.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ArchiveUris.java new file mode 100644 index 00000000000..a0f6955b59f --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ArchiveUris.java @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.configserver; + +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.TenantName; + +import java.net.URI; +import java.util.Map; + +/** + * @author freva + */ +public record ArchiveUris(Map<TenantName, URI> tenantArchiveUris, Map<CloudAccount, URI> accountArchiveUris) { + public static final ArchiveUris EMPTY = new ArchiveUris(Map.of(), Map.of()); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java index 99b2968a43f..2b35334e14b 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java @@ -6,6 +6,8 @@ import com.yahoo.slime.SlimeUtils; import java.util.stream.Stream; +import static java.nio.charset.StandardCharsets.UTF_8; + /** * An exception due to server error, a bad request, or similar. * @@ -56,7 +58,7 @@ public class ConfigServerException extends RuntimeException { ErrorCode code = Stream.of(ErrorCode.values()) .filter(value -> value.name().equals(codeName)) .findAny().orElse(ErrorCode.INCOMPLETE_RESPONSE); - String message = root.field("message").valid() ? root.field("message").asString() : "(no message)"; + String message = root.field("message").valid() ? root.field("message").asString() : new String(body, UTF_8); return new ConfigServerException(code, message, context); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeFilter.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeFilter.java index 7b209d231c4..796ce5da449 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeFilter.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeFilter.java @@ -20,14 +20,16 @@ public class NodeFilter { private final boolean includeDeprovisioned; private final Set<Node.State> states; private final Set<HostName> hostnames; + private final Set<HostName> parentHostnames; private final Set<ApplicationId> applications; private NodeFilter(boolean includeDeprovisioned, Set<Node.State> states, Set<HostName> hostnames, - Set<ApplicationId> applications) { + Set<HostName> parentHostnames, Set<ApplicationId> applications) { this.includeDeprovisioned = includeDeprovisioned; // Uses Guava Set to preserve insertion order this.states = ImmutableSet.copyOf(Objects.requireNonNull(states)); this.hostnames = ImmutableSet.copyOf(Objects.requireNonNull(hostnames)); + this.parentHostnames = ImmutableSet.copyOf(Objects.requireNonNull(parentHostnames)); this.applications = ImmutableSet.copyOf(Objects.requireNonNull(applications)); if (!includeDeprovisioned && states.contains(Node.State.deprovisioned)) { throw new IllegalArgumentException("Must include deprovisioned nodes when matching deprovisioned state"); @@ -46,12 +48,16 @@ public class NodeFilter { return hostnames; } + public Set<HostName> parentHostnames() { + return parentHostnames; + } + public Set<ApplicationId> applications() { return applications; } public NodeFilter includeDeprovisioned(boolean includeDeprovisioned) { - return new NodeFilter(includeDeprovisioned, states, hostnames, applications); + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); } public NodeFilter states(Node.State... states) { @@ -59,7 +65,7 @@ public class NodeFilter { } public NodeFilter states(Set<Node.State> states) { - return new NodeFilter(includeDeprovisioned, states, hostnames, applications); + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); } public NodeFilter hostnames(HostName... hostnames) { @@ -67,7 +73,15 @@ public class NodeFilter { } public NodeFilter hostnames(Set<HostName> hostnames) { - return new NodeFilter(includeDeprovisioned, states, hostnames, applications); + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); + } + + public NodeFilter parentHostnames(HostName... parentHostnames) { + return parentHostnames(ImmutableSet.copyOf(parentHostnames)); + } + + public NodeFilter parentHostnames(Set<HostName> parentHostnames) { + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); } public NodeFilter applications(ApplicationId... applications) { @@ -75,12 +89,12 @@ public class NodeFilter { } public NodeFilter applications(Set<ApplicationId> applications) { - return new NodeFilter(includeDeprovisioned, states, hostnames, applications); + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); } /** A filter which matches all nodes, except deprovisioned ones */ public static NodeFilter all() { - return new NodeFilter(false, Set.of(), Set.of(), Set.of()); + return new NodeFilter(false, Set.of(), Set.of(), Set.of(), Set.of()); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java index 1768de8d012..4c5a67626ea 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java @@ -5,15 +5,10 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveUriUpdate; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.ApplicationPatch; -import javax.ws.rs.HeaderParam; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import java.net.URI; import java.util.List; import java.util.Map; @@ -48,14 +43,11 @@ public interface NodeRepository { /** Get node statistics such as cost and load from given zone */ NodeRepoStats getStats(ZoneId zone); - /** Get all archive URLs found in zone */ - Map<TenantName, URI> getArchiveUris(ZoneId zone); + /** Get all archive URIs found in zone */ + ArchiveUris getArchiveUris(ZoneId zone); - /** Update archive URL for given tenant */ - void setArchiveUri(ZoneId zone, TenantName tenantName, URI archiveUri); - - /** Remove archive URL for given tenant */ - void removeArchiveUri(ZoneId zone, TenantName tenantName); + /** Update some archive URI in the given zone */ + void updateArchiveUri(ZoneId zone, ArchiveUriUpdate archiveUriUpdate); /** Upgrade all nodes of given type to a new version */ void upgrade(ZoneId zone, NodeType type, Version version, boolean allowDowngrade); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MockVpcEndpointService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MockVpcEndpointService.java index 563b343dab5..39975138140 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MockVpcEndpointService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MockVpcEndpointService.java @@ -18,7 +18,7 @@ import java.util.concurrent.atomic.AtomicBoolean; public class MockVpcEndpointService implements VpcEndpointService { public final AtomicBoolean enabled = new AtomicBoolean(); - public final Map<RecordName, State> outcomes = new ConcurrentHashMap<>(); + public final Map<RecordName, ChallengeState> outcomes = new ConcurrentHashMap<>(); private final Clock clock; private final NameService nameService; @@ -36,20 +36,20 @@ public class MockVpcEndpointService implements VpcEndpointService { "service-id", account, clock.instant(), - State.pending); + ChallengeState.pending); return Optional.ofNullable(enabled.get() && nameService.findRecords(Type.TXT, challenge.name()).isEmpty() ? challenge : null); } @Override - public synchronized State process(DnsChallenge challenge) { + public synchronized ChallengeState process(DnsChallenge challenge) { if (outcomes.containsKey(challenge.name())) return outcomes.get(challenge.name()); if (nameService.findRecords(Type.TXT, challenge.name()).isEmpty()) throw new RuntimeException("No TXT record found for " + challenge.name()); - return State.done; + return ChallengeState.done; } @Override public synchronized List<VpcEndpoint> getConnections(ClusterId cluster, Optional<CloudAccount> account) { - return List.of(new VpcEndpoint("endpoint-1", "available")); + return List.of(new VpcEndpoint("endpoint-1", "available", EndpointState.open)); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/VpcEndpointService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/VpcEndpointService.java index 74459792987..a3ee7681e2a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/VpcEndpointService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/VpcEndpointService.java @@ -17,7 +17,7 @@ public interface VpcEndpointService { /** Create a TXT record with this name and token, and then complete the challenge. */ record DnsChallenge(RecordName name, RecordData data, ClusterId clusterId, String serviceId, - Optional<CloudAccount> account, Instant createdAt, State state) { + Optional<CloudAccount> account, Instant createdAt, ChallengeState state) { public DnsChallenge { requireNonNull(name, "name must be non-null"); @@ -29,22 +29,24 @@ public interface VpcEndpointService { requireNonNull(state, "state must be non-null"); } - public DnsChallenge withState(State state) { + public DnsChallenge withState(ChallengeState state) { return new DnsChallenge(name, data, clusterId, serviceId, account, createdAt, state); } } - enum State { pending, ready, running, done } + enum ChallengeState { pending, ready, running, done } /** Sets the private DNS name for any VPC endpoint for the given cluster, potentially guarded by a challenge. */ Optional<DnsChallenge> setPrivateDns(DomainName privateDnsName, ClusterId clusterId, Optional<CloudAccount> account); /** Attempts to complete the challenge, and returns the updated challenge state. */ - State process(DnsChallenge challenge); + ChallengeState process(DnsChallenge challenge); /** A connection made to an endpoint service. */ - record VpcEndpoint(String endpointId, String state) { } + record VpcEndpoint(String endpointId, String stateString, EndpointState stateValue) { } + + enum EndpointState { pending, open, failed, closed } /** Lists all endpoints connected to an endpoint service (owned by account) for the given cluster. */ List<VpcEndpoint> getConnections(ClusterId cluster, Optional<CloudAccount> account); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ArchiveList.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ArchiveList.java index 274d07bfc3b..172523eb261 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ArchiveList.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ArchiveList.java @@ -16,6 +16,9 @@ public class ArchiveList { @JsonProperty("tenant") public String tenant; + @JsonProperty("account") + public String account; + @JsonProperty("uri") public String uri; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java index 70eaca5ce7e..4956dd475de 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java @@ -38,6 +38,9 @@ public interface ZoneRegistry { /** Returns whether cloudAccount in this system supports given zone */ boolean hasZone(ZoneId zoneId, CloudAccount cloudAccount); + /** Returns whether the given cloud account is an enclave */ + boolean isEnclave(CloudAccount cloudAccount); + /** Returns a list containing the id of all zones in this registry */ ZoneFilter zones(); diff --git a/controller-server/pom.xml b/controller-server/pom.xml index 1a8a68be9e0..64cc89c3321 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -125,7 +125,7 @@ <dependency> <groupId>commons-fileupload</groupId> <artifactId>commons-fileupload</artifactId> - <version>1.4</version> + <version>1.5</version> </dependency> <dependency> @@ -247,6 +247,14 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Xlint:-serial</arg> + <arg>-Xlint:-try</arg> + <arg>-Xlint:-processing</arg> + </compilerArgs> + </configuration> </plugin> </plugins> </build> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index a4bded314d9..a1534ebc533 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -526,8 +526,8 @@ public class ApplicationController { }; // Carry out deployment without holding the application lock. - DeploymentResult result = deploy(job.application(), applicationPackage, zone, platform, containerEndpoints, - endpointCertificateMetadata, run.isDryRun(), run.testerCertificate()); + DeploymentDataAndResult dataAndResult = deploy(job.application(), applicationPackage, zone, platform, containerEndpoints, + endpointCertificateMetadata, run.isDryRun(), run.testerCertificate()); // Record the quota usage for this application @@ -540,7 +540,7 @@ public class ApplicationController { ? NotificationSource.from(deployment) : revision.equals(lastRevision.get()) ? NotificationSource.from(applicationId) : null; if (source != null) { - List<String> warnings = Optional.ofNullable(result.log()) + List<String> warnings = Optional.ofNullable(dataAndResult.result().log()) .map(logs -> logs.stream() .filter(LogEntry::concernsPackage) .filter(log -> log.level().intValue() >= Level.WARNING.intValue()) @@ -558,9 +558,9 @@ public class ApplicationController { lockApplicationOrThrow(applicationId, application -> store(application.with(job.application().instance(), i -> i.withNewDeployment(zone, revision, platform, - clock.instant(), warningsFrom(result.log()), - quotaUsage)))); - return result; + clock.instant(), warningsFrom(dataAndResult.result().log()), + quotaUsage, dataAndResult.data().cloudAccount().orElse(CloudAccount.empty))))); + return dataAndResult.result(); } } @@ -618,7 +618,7 @@ public class ApplicationController { ApplicationPackageStream applicationPackage = new ApplicationPackageStream( () -> new ByteArrayInputStream(artifactRepository.getSystemApplicationPackage(application.id(), zone, version)) ); - return deploy(application.id(), applicationPackage, zone, version, Set.of(), Optional::empty, false, Optional.empty()); + return deploy(application.id(), applicationPackage, zone, version, Set.of(), Optional::empty, false, Optional.empty()).result(); } else { throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString()); } @@ -626,15 +626,18 @@ public class ApplicationController { /** Deploys the given tester application to the given zone. */ public DeploymentResult deployTester(TesterId tester, ApplicationPackageStream applicationPackage, ZoneId zone, Version platform) { - return deploy(tester.id(), applicationPackage, zone, platform, Set.of(), Optional::empty, false, Optional.empty()); + return deploy(tester.id(), applicationPackage, zone, platform, Set.of(), Optional::empty, false, Optional.empty()).result(); } - private DeploymentResult deploy(ApplicationId application, ApplicationPackageStream applicationPackage, - ZoneId zone, Version platform, Set<ContainerEndpoint> endpoints, - Supplier<Optional<EndpointCertificateMetadata>> endpointCertificateMetadata, - boolean dryRun, Optional<X509Certificate> testerCertificate) { + private record DeploymentDataAndResult(DeploymentData data, DeploymentResult result) {} + private DeploymentDataAndResult deploy(ApplicationId application, ApplicationPackageStream applicationPackage, + ZoneId zone, Version platform, Set<ContainerEndpoint> endpoints, + Supplier<Optional<EndpointCertificateMetadata>> endpointCertificateMetadata, + boolean dryRun, Optional<X509Certificate> testerCertificate) { DeploymentId deployment = new DeploymentId(application, zone); - try { + // Routing and metadata may have changed, so we need to refresh state after deployment, even if deployment fails. + interface CleanCloseable extends AutoCloseable { void close(); } + try (CleanCloseable postDeployment = () -> updateRoutingAndMeta(deployment, applicationPackage)) { Optional<DockerImage> dockerImageRepo = Optional.ofNullable( dockerImageRepoFlag .with(FetchVector.Dimension.ZONE_ID, zone.value()) @@ -662,26 +665,22 @@ public class ApplicationController { operatorCertificates = Stream.concat(operatorCertificates.stream(), testerCertificate.stream()).toList(); } Supplier<Optional<CloudAccount>> cloudAccount = () -> decideCloudAccountOf(deployment, applicationPackage.truncatedPackage().deploymentSpec()); - ConfigServer.PreparedApplication preparedApplication = - configServer.deploy(new DeploymentData(application, zone, applicationPackage::zipStream, platform, - endpoints, endpointCertificateMetadata, dockerImageRepo, domain, - deploymentQuota, tenantSecretStores, operatorCertificates, - cloudAccount, dryRun)); - - return preparedApplication.deploymentResult(); - } finally { - // Even if prepare fails, routing configuration may need to be updated - if ( ! application.instance().isTester()) { - controller.routing().of(deployment).configure(applicationPackage.truncatedPackage().deploymentSpec()); - if (zone.environment().isManuallyDeployed()) - controller.applications().applicationStore().putMeta(deployment, - clock.instant(), - applicationPackage.truncatedPackage().metaDataZip()); + DeploymentData deploymentData = new DeploymentData(application, zone, applicationPackage::zipStream, platform, + endpoints, endpointCertificateMetadata, dockerImageRepo, domain, + deploymentQuota, tenantSecretStores, operatorCertificates, cloudAccount, dryRun); + ConfigServer.PreparedApplication preparedApplication = configServer.deploy(deploymentData); - } + return new DeploymentDataAndResult(deploymentData, preparedApplication.deploymentResult()); } } + private void updateRoutingAndMeta(DeploymentId id, ApplicationPackageStream data) { + if (id.applicationId().instance().isTester()) return; + controller.routing().of(id).configure(data.truncatedPackage().deploymentSpec()); + if ( ! id.zoneId().environment().isManuallyDeployed()) return; + controller.applications().applicationStore().putMeta(id, clock.instant(), data.truncatedPackage().metaDataZip()); + } + public Optional<CloudAccount> decideCloudAccountOf(DeploymentId deployment, DeploymentSpec spec) { ZoneId zoneId = deployment.zoneId(); Optional<CloudAccount> requestedAccount = spec.instance(deployment.applicationId().instance()) @@ -892,16 +891,17 @@ public class ApplicationController { */ private Optional<LockedApplication> deactivate(ApplicationId instanceId, ZoneId zone, Optional<LockedApplication> application) { DeploymentId id = new DeploymentId(instanceId, zone); - try { - configServer.deactivate(id); - } finally { + interface CleanCloseable extends AutoCloseable { void close(); } + try (CleanCloseable postDeactivation = () -> { application.ifPresent(app -> controller.routing().of(id).configure(app.get().deploymentSpec())); if (id.zoneId().environment().isManuallyDeployed()) applicationStore.putMetaTombstone(id, clock.instant()); - if (!id.zoneId().environment().isTest()) + if ( ! id.zoneId().environment().isTest()) controller.notificationsDb().removeNotifications(NotificationSource.from(id)); + }) { + configServer.deactivate(id); + return application.map(app -> app.with(instanceId.instance(), instance -> instance.withoutDeploymentIn(id.zoneId()))); } - return application.map(app -> app.with(instanceId.instance(), instance -> instance.withoutDeploymentIn(id.zoneId()))); } public DeploymentTrigger deploymentTrigger() { return deploymentTrigger; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java index d66d1491f73..14bd537a056 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.zone.ZoneId; @@ -63,16 +64,16 @@ public class Instance { this.change = Objects.requireNonNull(change, "change cannot be null"); } - public Instance withNewDeployment(ZoneId zone, RevisionId revision, Version version, - Instant instant, Map<DeploymentMetrics.Warning, Integer> warnings, QuotaUsage quotaUsage) { + public Instance withNewDeployment(ZoneId zone, RevisionId revision, Version version, Instant instant, + Map<DeploymentMetrics.Warning, Integer> warnings, QuotaUsage quotaUsage, CloudAccount cloudAccount) { // Use info from previous deployment if available, otherwise create a new one. - Deployment previousDeployment = deployments.getOrDefault(zone, new Deployment(zone, revision, + Deployment previousDeployment = deployments.getOrDefault(zone, new Deployment(zone, cloudAccount, revision, version, instant, DeploymentMetrics.none, DeploymentActivity.none, QuotaUsage.none, OptionalDouble.empty())); - Deployment newDeployment = new Deployment(zone, revision, version, instant, + Deployment newDeployment = new Deployment(zone, cloudAccount, revision, version, instant, previousDeployment.metrics().with(warnings), previousDeployment.activity(), quotaUsage, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java index 2e4afb4e004..6d4fddfbc0a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.component.Version; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; @@ -18,6 +19,7 @@ import java.util.OptionalDouble; public class Deployment { private final ZoneId zone; + private final CloudAccount cloudAccount; private final RevisionId revision; private final Version version; private final Instant deployTime; @@ -26,9 +28,10 @@ public class Deployment { private final QuotaUsage quota; private final OptionalDouble cost; - public Deployment(ZoneId zone, RevisionId revision, Version version, Instant deployTime, - DeploymentMetrics metrics, DeploymentActivity activity, QuotaUsage quota, OptionalDouble cost) { + public Deployment(ZoneId zone, CloudAccount cloudAccount, RevisionId revision, Version version, Instant deployTime, + DeploymentMetrics metrics, DeploymentActivity activity, QuotaUsage quota, OptionalDouble cost) { this.zone = Objects.requireNonNull(zone, "zone cannot be null"); + this.cloudAccount = Objects.requireNonNull(cloudAccount, "cloudAccount cannot be null"); this.revision = Objects.requireNonNull(revision, "revision cannot be null"); this.version = Objects.requireNonNull(version, "version cannot be null"); this.deployTime = Objects.requireNonNull(deployTime, "deployTime cannot be null"); @@ -41,6 +44,9 @@ public class Deployment { /** Returns the zone this was deployed to */ public ZoneId zone() { return zone; } + /** Returns the cloud account this was deployed to */ + public CloudAccount cloudAccount() { return cloudAccount; } + /** Returns the deployed application revision */ public RevisionId revision() { return revision; } @@ -65,26 +71,22 @@ public class Deployment { public OptionalDouble cost() { return cost; } public Deployment recordActivityAt(Instant instant) { - return new Deployment(zone, revision, version, deployTime, metrics, + return new Deployment(zone, cloudAccount, revision, version, deployTime, metrics, activity.recordAt(instant, metrics), quota, cost); } public Deployment withMetrics(DeploymentMetrics metrics) { - return new Deployment(zone, revision, version, deployTime, metrics, activity, quota, cost); - } - - public Deployment withQuota(QuotaUsage quota) { - return new Deployment(zone, revision, version, deployTime, metrics, activity, quota, cost); + return new Deployment(zone, cloudAccount, revision, version, deployTime, metrics, activity, quota, cost); } public Deployment withCost(double cost) { if (this.cost.isPresent() && Double.compare(this.cost.getAsDouble(), cost) == 0) return this; - return new Deployment(zone, revision, version, deployTime, metrics, activity, quota, OptionalDouble.of(cost)); + return new Deployment(zone, cloudAccount, revision, version, deployTime, metrics, activity, quota, OptionalDouble.of(cost)); } public Deployment withoutCost() { if (cost.isEmpty()) return this; - return new Deployment(zone, revision, version, deployTime, metrics, activity, quota, OptionalDouble.empty()); + return new Deployment(zone, cloudAccount, revision, version, deployTime, metrics, activity, quota, OptionalDouble.empty()); } @Override @@ -93,6 +95,7 @@ public class Deployment { if (o == null || getClass() != o.getClass()) return false; Deployment that = (Deployment) o; return zone.equals(that.zone) && + cloudAccount.equals(that.cloudAccount) && revision.equals(that.revision) && version.equals(that.version) && deployTime.equals(that.deployTime) && @@ -104,7 +107,7 @@ public class Deployment { @Override public int hashCode() { - return Objects.hash(zone, revision, version, deployTime, metrics, activity, quota, cost); + return Objects.hash(zone, cloudAccount, revision, version, deployTime, metrics, activity, quota, cost); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java index ac32fe5799d..962bd144a21 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java @@ -1,18 +1,22 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.archive; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBuckets; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; +import com.yahoo.vespa.hosted.controller.api.integration.archive.TenantManagedArchiveBucket; +import com.yahoo.vespa.hosted.controller.api.integration.archive.VespaManagedArchiveBucket; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.net.URI; -import java.util.HashSet; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -23,81 +27,100 @@ import java.util.stream.Collectors; */ public class CuratorArchiveBucketDb { + private static final Duration ENCLAVE_BUCKET_CACHE_LIFETIME = Duration.ofMinutes(60); + /** * Archive URIs are often requested because they are returned in /application/v4 API. Since they * never change, it's safe to cache them and only update on misses */ private final Map<ZoneId, Map<TenantName, String>> archiveUriCache = new ConcurrentHashMap<>(); + private final Map<ZoneId, Map<CloudAccount, TenantManagedArchiveBucket>> tenantArchiveCache = new ConcurrentHashMap<>(); private final ArchiveService archiveService; private final CuratorDb curatorDb; + private final Clock clock; public CuratorArchiveBucketDb(Controller controller) { this.archiveService = controller.serviceRegistry().archiveService(); this.curatorDb = controller.curator(); + this.clock = controller.clock(); } public Optional<URI> archiveUriFor(ZoneId zoneId, TenantName tenant, boolean createIfMissing) { return getBucketNameFromCache(zoneId, tenant) - .or(() -> findAndUpdateArchiveUriCache(zoneId, tenant, buckets(zoneId))) .or(() -> createIfMissing ? Optional.of(assignToBucket(zoneId, tenant)) : Optional.empty()) - .map(bucketName -> archiveService.bucketURI(zoneId, bucketName, tenant)); + .map(bucketName -> archiveService.bucketURI(zoneId, bucketName)); + } + + public Optional<URI> archiveUriFor(ZoneId zoneId, CloudAccount account, boolean searchIfMissing) { + Instant updatedAfter = searchIfMissing ? clock.instant().minus(ENCLAVE_BUCKET_CACHE_LIFETIME) : Instant.MIN; + return getBucketNameFromCache(zoneId, account, updatedAfter) + .or(() -> { + if (!searchIfMissing) return Optional.empty(); + try (var lock = curatorDb.lockArchiveBuckets(zoneId)) { + ArchiveBuckets archiveBuckets = buckets(zoneId); + updateArchiveUriCache(zoneId, archiveBuckets); + + return getBucketNameFromCache(zoneId, account, updatedAfter) + .or(() -> archiveService.findEnclaveArchiveBucket(zoneId, account) + .map(bucketName -> { + var bucket = new TenantManagedArchiveBucket(bucketName, account, clock.instant()); + ArchiveBuckets updated = archiveBuckets.with(bucket); + curatorDb.writeArchiveBuckets(zoneId, updated); + updateArchiveUriCache(zoneId, updated); + return bucket; + })); + } + }) + .map(TenantManagedArchiveBucket::bucketName) + .map(bucketName -> archiveService.bucketURI(zoneId, bucketName)); } private String assignToBucket(ZoneId zoneId, TenantName tenant) { try (var lock = curatorDb.lockArchiveBuckets(zoneId)) { - Set<ArchiveBucket> zoneBuckets = new HashSet<>(buckets(zoneId)); + ArchiveBuckets archiveBuckets = buckets(zoneId); + updateArchiveUriCache(zoneId, archiveBuckets); - return findAndUpdateArchiveUriCache(zoneId, tenant, zoneBuckets) // Some other thread might have assigned it before we grabbed the lock + return getBucketNameFromCache(zoneId, tenant) // Some other thread might have assigned it before we grabbed the lock .orElseGet(() -> { // If not, find an existing bucket with space - Optional<ArchiveBucket> unfilledBucket = zoneBuckets.stream() + VespaManagedArchiveBucket bucketToAssignTo = archiveBuckets.vespaManaged().stream() .filter(bucket -> archiveService.canAddTenantToBucket(zoneId, bucket)) - .findAny(); - - // And place the tenant in that bucket. - if (unfilledBucket.isPresent()) { - var unfilled = unfilledBucket.get(); - - zoneBuckets.remove(unfilled); - zoneBuckets.add(unfilled.withTenant(tenant)); - curatorDb.writeArchiveBuckets(zoneId, zoneBuckets); + .findAny() + // Or create a new one + .orElseGet(() -> archiveService.createArchiveBucketFor(zoneId)); - return unfilled.bucketName(); - } + ArchiveBuckets updated = archiveBuckets.with(bucketToAssignTo.withTenant(tenant)); + curatorDb.writeArchiveBuckets(zoneId, updated); + updateArchiveUriCache(zoneId, updated); - // We'll have to create a new bucket - var newBucket = archiveService.createArchiveBucketFor(zoneId).withTenant(tenant); - zoneBuckets.add(newBucket); - curatorDb.writeArchiveBuckets(zoneId, zoneBuckets); - updateArchiveUriCache(zoneId, zoneBuckets); - return newBucket.bucketName(); + return bucketToAssignTo.bucketName(); }); } } - public Set<ArchiveBucket> buckets(ZoneId zoneId) { + public ArchiveBuckets buckets(ZoneId zoneId) { return curatorDb.readArchiveBuckets(zoneId); } - private Optional<String> findAndUpdateArchiveUriCache(ZoneId zoneId, TenantName tenant, Set<ArchiveBucket> zoneBuckets) { - Optional<String> bucketName = zoneBuckets.stream() - .filter(bucket -> bucket.tenants().contains(tenant)) - .findAny() - .map(ArchiveBucket::bucketName); - if (bucketName.isPresent()) updateArchiveUriCache(zoneId, zoneBuckets); - return bucketName; - } - private Optional<String> getBucketNameFromCache(ZoneId zoneId, TenantName tenantName) { return Optional.ofNullable(archiveUriCache.get(zoneId)).map(map -> map.get(tenantName)); } - private void updateArchiveUriCache(ZoneId zoneId, Set<ArchiveBucket> zoneBuckets) { - Map<TenantName, String> bucketNameByTenant = zoneBuckets.stream() - .flatMap(bucket -> bucket.tenants().stream() - .map(tenant -> Map.entry(tenant, bucket.bucketName()))) + private Optional<TenantManagedArchiveBucket> getBucketNameFromCache(ZoneId zoneId, CloudAccount cloudAccount, Instant updatedAfter) { + return Optional.ofNullable(tenantArchiveCache.get(zoneId)) + .map(map -> map.get(cloudAccount)) + .filter(bucket -> bucket.updatedAt().isAfter(updatedAfter)); + } + + private void updateArchiveUriCache(ZoneId zoneId, ArchiveBuckets archiveBuckets) { + Map<TenantName, String> bucketNameByTenant = archiveBuckets.vespaManaged().stream() + .flatMap(bucket -> bucket.tenants().stream().map(tenant -> Map.entry(tenant, bucket.bucketName()))) .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); archiveUriCache.put(zoneId, bucketNameByTenant); + + Map<CloudAccount, TenantManagedArchiveBucket> bucketByAccount = archiveBuckets.tenantManaged().stream() + .collect(Collectors.toUnmodifiableMap(TenantManagedArchiveBucket::cloudAccount, bucket -> bucket)); + tenantArchiveCache.put(zoneId, bucketByAccount); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index 6a493f3f5ed..65320a25984 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -309,7 +309,7 @@ public class AthenzFacade implements AccessControl { } private boolean lookupAccess(AccessTuple t) { - boolean result = zmsClient.hasAccess(AthenzResourceName.fromString(t.resource), t.action, t.identity); + boolean result = ztsClient.hasAccess(AthenzResourceName.fromString(t.resource), t.action, t.identity); log("getAccess(action=%s, resource=%s, principal=%s) = %b", t.action, t.resource, t.identity, result); return result; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java index eed4fd0245d..b2ed0941c8e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java @@ -1,12 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.google.common.collect.Maps; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.archive.CuratorArchiveBucketDb; @@ -17,11 +15,8 @@ import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.time.Duration; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; -import static java.util.stream.Collectors.groupingBy; - /** * Update archive access permissions with roles from tenants * @@ -48,7 +43,7 @@ public class ArchiveAccessMaintainer extends ControllerMaintainer { protected double maintain() { // Count buckets - so we can alert if we get close to the AWS account limit of 1000 zoneRegistry.zonesIncludingSystem().all().zones().forEach(z -> - metric.set(bucketCountMetricName, archiveBucketDb.buckets(z.getVirtualId()).size(), + metric.set(bucketCountMetricName, archiveBucketDb.buckets(z.getVirtualId()).vespaManaged().size(), metric.createContext(Map.of( "zone", z.getVirtualId().value(), "cloud", z.getCloudName().value())))); @@ -57,7 +52,7 @@ public class ArchiveAccessMaintainer extends ControllerMaintainer { ZoneId zoneId = z.getVirtualId(); try { var tenantArchiveAccessRoles = cloudTenantArchiveExternalAccessRoles(); - var buckets = archiveBucketDb.buckets(zoneId); + var buckets = archiveBucketDb.buckets(zoneId).vespaManaged(); archiveService.updatePolicies(zoneId, buckets, tenantArchiveAccessRoles); } catch (Exception e) { throw new RuntimeException("Failed to maintain archive access in " + zoneId.value(), e); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java index 0c8a50fa821..518027f8099 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java @@ -1,22 +1,26 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveUriUpdate; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ArchiveUris; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.archive.CuratorArchiveBucketDb; import com.yahoo.yolean.Exceptions; -import java.net.URI; import java.time.Duration; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.logging.Level; +import java.util.stream.Stream; /** * Updates archive URIs for tenants in all zones. @@ -30,25 +34,31 @@ public class ArchiveUriUpdater extends ControllerMaintainer { private final ApplicationController applications; private final NodeRepository nodeRepository; private final CuratorArchiveBucketDb archiveBucketDb; + private final ZoneRegistry zoneRegistry; public ArchiveUriUpdater(Controller controller, Duration interval) { super(controller, interval); this.applications = controller.applications(); this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); this.archiveBucketDb = controller.archiveBucketDb(); + this.zoneRegistry = controller.zoneRegistry(); } @Override protected double maintain() { Map<ZoneId, Set<TenantName>> tenantsByZone = new HashMap<>(); + Map<ZoneId, Set<CloudAccount>> accountsByZone = new HashMap<>(); - controller().zoneRegistry().zonesIncludingSystem().reachable().zones().forEach( - z -> tenantsByZone.put(z.getVirtualId(), new HashSet<>(INFRASTRUCTURE_TENANTS))); + controller().zoneRegistry().zonesIncludingSystem().reachable().zones().forEach(zone -> { + tenantsByZone.put(zone.getVirtualId(), new HashSet<>(INFRASTRUCTURE_TENANTS)); + accountsByZone.put(zone.getVirtualId(), new HashSet<>()); + }); for (var application : applications.asList()) { for (var instance : application.instances().values()) { for (var deployment : instance.deployments().values()) { - tenantsByZone.get(deployment.zone()).add(instance.id().tenant()); + if (zoneRegistry.isEnclave(deployment.cloudAccount())) accountsByZone.get(deployment.zone()).add(deployment.cloudAccount()); + else tenantsByZone.get(deployment.zone()).add(instance.id().tenant()); } } } @@ -56,17 +66,31 @@ public class ArchiveUriUpdater extends ControllerMaintainer { int failures = 0; for (ZoneId zone : tenantsByZone.keySet()) { try { - Map<TenantName, URI> zoneArchiveUris = nodeRepository.getArchiveUris(zone); + ArchiveUris zoneArchiveUris = nodeRepository.getArchiveUris(zone); - for (TenantName tenant : tenantsByZone.get(zone)) { - archiveBucketDb.archiveUriFor(zone, tenant, true) - .filter(uri -> !uri.equals(zoneArchiveUris.get(tenant))) - .ifPresent(uri -> nodeRepository.setArchiveUri(zone, tenant, uri)); - } - - zoneArchiveUris.keySet().stream() - .filter(tenant -> !tenantsByZone.get(zone).contains(tenant)) - .forEach(tenant -> nodeRepository.removeArchiveUri(zone, tenant)); + Stream.of( + // Tenant URIs that need to be added or updated + tenantsByZone.get(zone).stream() + .flatMap(tenant -> archiveBucketDb.archiveUriFor(zone, tenant, true) + .filter(uri -> !uri.equals(zoneArchiveUris.tenantArchiveUris().get(tenant))) + .map(uri -> ArchiveUriUpdate.setArchiveUriFor(tenant, uri)) + .stream()), + // Account URIs that need to be added or updated + accountsByZone.get(zone).stream() + .flatMap(account -> archiveBucketDb.archiveUriFor(zone, account, true) + .filter(uri -> !uri.equals(zoneArchiveUris.accountArchiveUris().get(account))) + .map(uri -> ArchiveUriUpdate.setArchiveUriFor(account, uri)) + .stream()), + // Tenant URIs that need to be deleted + zoneArchiveUris.tenantArchiveUris().keySet().stream() + .filter(tenant -> !tenantsByZone.get(zone).contains(tenant)) + .map(ArchiveUriUpdate::deleteArchiveUriFor), + // Account URIs that need to be deleted + zoneArchiveUris.accountArchiveUris().keySet().stream() + .filter(account -> !accountsByZone.get(zone).contains(account)) + .map(ArchiveUriUpdate::deleteArchiveUriFor)) + .flatMap(s -> s) + .forEach(update -> nodeRepository.updateArchiveUri(zone, update)); } catch (Exception e) { log.log(Level.WARNING, "Failed to update archive URI in " + zone + ". Retrying in " + interval() + ". Error: " + Exceptions.toMessageString(e)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java index cf9db1517a0..f6029eade37 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java @@ -1,6 +1,5 @@ package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; @@ -9,7 +8,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeReposi import com.yahoo.yolean.Exceptions; import java.time.Duration; -import java.util.stream.Stream; +import java.util.Collection; /** * This pulls application deployment information from the node repo on all config servers, @@ -28,29 +27,35 @@ public class DeploymentInfoMaintainer extends ControllerMaintainer { @Override protected double maintain() { - controller().applications().asList().stream() - .flatMap(this::mapApplicationToInstances) - .flatMap(this::mapInstanceToDeployments) - .forEach(this::updateDeploymentInfo); - return 1.0; - } - - private Stream<Instance> mapApplicationToInstances(Application application) { - return application.instances().values().stream(); + int attempts = 0; + int failures = 0; + for (var application : controller().applications().asList()) { + for (var instance : application.instances().values()) { + for (var deployment : instanceDeployments(instance)) { + attempts++; + if ( ! updateDeploymentInfo(deployment)) + failures++; + } + } + } + return asSuccessFactor(attempts, failures); } - private Stream<DeploymentId> mapInstanceToDeployments(Instance instance) { + private Collection<DeploymentId> instanceDeployments(Instance instance) { return instance.deployments().keySet().stream() .filter(zoneId -> !zoneId.environment().isTest()) - .map(zoneId -> new DeploymentId(instance.id(), zoneId)); + .map(zoneId -> new DeploymentId(instance.id(), zoneId)) + .toList(); } - private void updateDeploymentInfo(DeploymentId id) { + private boolean updateDeploymentInfo(DeploymentId id) { try { controller().applications().deploymentInfo().put(id, nodeRepository.getApplication(id.zoneId(), id.applicationId())); + return true; } catch (ConfigServerException e) { log.info("Could not retrieve deployment info for " + id + ": " + Exceptions.toMessageString(e)); + return false; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index d49cb244e47..82b3141e503 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -24,7 +24,6 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.OptionalInt; import java.util.Random; import java.util.Set; import java.util.function.UnaryOperator; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java index 45b3e4ef5dd..da0fa890960 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java @@ -192,17 +192,15 @@ public class VcmrMaintainer extends ControllerMaintainer { } if (shouldRetire(changeRequest, hostAction)) { - if (!node.wantToRetire()) { + if (!wantToRetireRecursive(zoneId, node)) { LOG.info(Text.format("Retiring %s due to %s", node.hostname().value(), changeRequest.getChangeRequestSource().getId())); // TODO: Remove try/catch once retirement is stabilized try { setWantToRetire(zoneId, node, true); } catch (Exception e) { LOG.warning("Failed to retire host " + node.hostname() + ": " + Exceptions.toMessageString(e)); - // Check if retirement actually failed - if (!nodeRepository.getNode(zoneId, node.hostname().value()).wantToRetire()) { - return hostAction; - } + // Will retry next maintenance run + return hostAction; } } return hostAction.withState(State.RETIRING); @@ -225,6 +223,13 @@ public class VcmrMaintainer extends ControllerMaintainer { return hostAction; } + // Determines if a host and all its children are retiring + private boolean wantToRetireRecursive(ZoneId zoneId, Node node) { + var children = nodeRepository.list(zoneId, NodeFilter.all().parentHostnames(node.hostname())); + return node.wantToRetire() && + children.stream().allMatch(Node::wantToRetire); + } + // Dirty host iff the parked host was retired by this maintainer private void recycleNode(ZoneId zoneId, Node node, HostAction hostAction) { if (hostAction.getState() == State.RETIRED && diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java index 1c76f58a6b2..82dc333d178 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java @@ -9,6 +9,7 @@ import com.yahoo.text.Text; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; import com.yahoo.vespa.hosted.controller.api.integration.organization.MailerException; @@ -76,7 +77,7 @@ public class Notifier { } private boolean dispatchEnabled(NotificationSource source) { - return Flags.NOTIFICATION_DISPATCH_FLAG.bindTo(flagSource) + return PermanentFlags.NOTIFICATION_DISPATCH_FLAG.bindTo(flagSource) .with(FetchVector.Dimension.TENANT_ID, source.tenant().value()) .value(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 2a9724bb911..ee12c9957b1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; @@ -103,6 +104,7 @@ public class ApplicationSerializer { // Deployment fields private static final String zoneField = "zone"; + private static final String cloudAccountField = "cloudAccount"; private static final String environmentField = "environment"; private static final String regionField = "region"; private static final String deployTimeField = "deployTime"; @@ -202,6 +204,7 @@ public class ApplicationSerializer { private void deploymentToSlime(Deployment deployment, Cursor object) { zoneIdToSlime(deployment.zone(), object.setObject(zoneField)); + if (!deployment.cloudAccount().isUnspecified()) object.setString(cloudAccountField, deployment.cloudAccount().value()); object.setString(versionField, deployment.version().toString()); object.setLong(deployTimeField, deployment.at().toEpochMilli()); toSlime(deployment.revision(), object.setObject(applicationPackageRevisionField)); @@ -409,6 +412,7 @@ public class ApplicationSerializer { private Deployment deploymentFromSlime(Inspector deploymentObject, ApplicationId id) { ZoneId zone = zoneIdFromSlime(deploymentObject.field(zoneField)); return new Deployment(zone, + SlimeUtils.optionalString(deploymentObject.field(cloudAccountField)).map(CloudAccount::from).orElse(CloudAccount.empty), revisionFromSlime(deploymentObject.field(applicationPackageRevisionField), new JobId(id, JobType.deploymentTo(zone))), Version.fromString(deploymentObject.field(versionField).asString()), SlimeUtils.instant(deploymentObject.field(deployTimeField)), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializer.java index a4c7c50085c..f40193510ce 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializer.java @@ -1,12 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.TenantName; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBuckets; +import com.yahoo.vespa.hosted.controller.api.integration.archive.TenantManagedArchiveBucket; +import com.yahoo.vespa.hosted.controller.api.integration.archive.VespaManagedArchiveBucket; import java.util.Set; import java.util.stream.Collectors; @@ -25,46 +28,64 @@ public class ArchiveBucketsSerializer { // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - private final static String bucketsFieldName = "buckets"; + private final static String vespaManagedBucketsFieldName = "buckets"; + private final static String tenantManagedBucketsFieldName = "tenantManagedBuckets"; private final static String bucketNameFieldName = "bucketName"; private final static String keyArnFieldName = "keyArn"; private final static String tenantsFieldName = "tenantIds"; + private final static String accountFieldName = "account"; + private final static String updatedAtFieldName = "updatedAt"; - public static Slime toSlime(Set<ArchiveBucket> archiveBuckets) { + public static Slime toSlime(ArchiveBuckets archiveBuckets) { Slime slime = new Slime(); Cursor rootObject = slime.setObject(); - Cursor bucketsArray = rootObject.setArray(bucketsFieldName); - archiveBuckets.forEach(bucket -> { - Cursor cursor = bucketsArray.addObject(); - cursor.setString(bucketNameFieldName, bucket.bucketName()); - cursor.setString(keyArnFieldName, bucket.keyArn()); - Cursor tenants = cursor.setArray(tenantsFieldName); - bucket.tenants().forEach(tenantName -> tenants.addString(tenantName.value())); - } - ); + Cursor vespaBucketsArray = rootObject.setArray(vespaManagedBucketsFieldName); + archiveBuckets.vespaManaged().forEach(bucket -> { + Cursor cursor = vespaBucketsArray.addObject(); + cursor.setString(bucketNameFieldName, bucket.bucketName()); + cursor.setString(keyArnFieldName, bucket.keyArn()); + Cursor tenants = cursor.setArray(tenantsFieldName); + bucket.tenants().forEach(tenantName -> tenants.addString(tenantName.value())); + }); + + Cursor tenantBucketsArray = rootObject.setArray(tenantManagedBucketsFieldName); + archiveBuckets.tenantManaged().forEach(bucket -> { + Cursor cursor = tenantBucketsArray.addObject(); + cursor.setString(bucketNameFieldName, bucket.bucketName()); + cursor.setString(accountFieldName, bucket.cloudAccount().value()); + cursor.setLong(updatedAtFieldName, bucket.updatedAt().toEpochMilli()); + }); return slime; } - public static Set<ArchiveBucket> fromSlime(Inspector inspector) { - return SlimeUtils.entriesStream(inspector.field(bucketsFieldName)) - .map(ArchiveBucketsSerializer::fromInspector) - .collect(Collectors.toUnmodifiableSet()); + public static ArchiveBuckets fromSlime(Slime slime) { + Inspector inspector = slime.get(); + return new ArchiveBuckets( + SlimeUtils.entriesStream(inspector.field(vespaManagedBucketsFieldName)) + .map(ArchiveBucketsSerializer::vespaManagedArchiveBucketFromInspector) + .collect(Collectors.toUnmodifiableSet()), + SlimeUtils.entriesStream(inspector.field(tenantManagedBucketsFieldName)) + .map(ArchiveBucketsSerializer::tenantManagedArchiveBucketFromInspector) + .collect(Collectors.toUnmodifiableSet())); } - private static ArchiveBucket fromInspector(Inspector inspector) { + private static VespaManagedArchiveBucket vespaManagedArchiveBucketFromInspector(Inspector inspector) { Set<TenantName> tenants = SlimeUtils.entriesStream(inspector.field(tenantsFieldName)) .map(i -> TenantName.from(i.asString())) .collect(Collectors.toUnmodifiableSet()); - return new ArchiveBucket( + return new VespaManagedArchiveBucket( inspector.field(bucketNameFieldName).asString(), inspector.field(keyArnFieldName).asString()) .withTenants(tenants); } - public static Set<ArchiveBucket> fromJsonString(String zkData) { - return fromSlime(SlimeUtils.jsonToSlime(zkData).get()); + private static TenantManagedArchiveBucket tenantManagedArchiveBucketFromInspector(Inspector inspector) { + return new TenantManagedArchiveBucket( + inspector.field(bucketNameFieldName).asString(), + CloudAccount.from(inspector.field(accountFieldName).asString()), + SlimeUtils.instant(inspector.field(updatedAtFieldName))); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index f4980073d6c..d4e6d7af4b4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -19,7 +19,7 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBuckets; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; @@ -632,12 +632,12 @@ public class CuratorDb { // -------------- Archive buckets ----------------------------------------- - public Set<ArchiveBucket> readArchiveBuckets(ZoneId zoneId) { - return curator.getData(archiveBucketsPath(zoneId)).map(String::new).map(ArchiveBucketsSerializer::fromJsonString) - .orElseGet(Set::of); + public ArchiveBuckets readArchiveBuckets(ZoneId zoneId) { + return readSlime(archiveBucketsPath(zoneId)).map(ArchiveBucketsSerializer::fromSlime) + .orElse(ArchiveBuckets.EMPTY); } - public void writeArchiveBuckets(ZoneId zoneid, Set<ArchiveBucket> archiveBuckets) { + public void writeArchiveBuckets(ZoneId zoneid, ArchiveBuckets archiveBuckets) { curator.set(archiveBucketsPath(zoneid), asJson(ArchiveBucketsSerializer.toSlime(archiveBuckets))); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/DnsChallengeSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/DnsChallengeSerializer.java index 2518fe48508..bb3b2c5035f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/DnsChallengeSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/DnsChallengeSerializer.java @@ -1,18 +1,14 @@ package com.yahoo.vespa.hosted.controller.persistence; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.DnsChallenge; -import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.State; +import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.ChallengeState; import java.time.Instant; @@ -53,17 +49,17 @@ class DnsChallengeSerializer { return uncheck(() -> SlimeUtils.toJsonBytes(slime)); } - private static State toState(String value) { + private static ChallengeState toState(String value) { return switch (value) { - case "pending" -> State.pending; - case "ready" -> State.ready; - case "running" -> State.running; - case "done" -> State.done; + case "pending" -> ChallengeState.pending; + case "ready" -> ChallengeState.ready; + case "running" -> ChallengeState.running; + case "done" -> ChallengeState.done; default -> throw new IllegalArgumentException("invalid serialized state: " + value); }; } - private static String toString(State state) { + private static String toString(ChallengeState state) { return switch (state) { case pending -> "pending"; case ready -> "ready"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 7fcad017569..b1df25c933b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -596,7 +596,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { var mergedAddress = updateTenantInfoAddress(inspector.field("address"), info.address()); var mergedInfo = info - .withName(getString(inspector.field("tenant").field("name"), info.name())) + .withName(getString(inspector.field("tenant").field("company"), info.name())) .withWebsite(getString(inspector.field("tenant").field("website"), info.website())) .withContact(mergedContact) .withAddress(mergedAddress); @@ -758,7 +758,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } private String getString(Inspector field, String defaultVale) { - return field.valid() ? field.asString().trim() : defaultVale; + var string = field.valid() ? field.asString().trim() : defaultVale; + if (string.length() > 512) throw new IllegalArgumentException("Input value too long"); + return string; } private SlimeJsonResponse updateTenantInfo(CloudTenant tenant, HttpRequest request) { @@ -1869,6 +1871,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { application.projectId().ifPresent(i -> response.setString("screwdriverId", String.valueOf(i))); + // TODO (freva): Get cloudAccount from deployment once all applications have redeployed once controller.applications().decideCloudAccountOf(deploymentId, application.deploymentSpec()).ifPresent(cloudAccount -> { Cursor enclave = response.setObject("enclave"); enclave.setString("cloudAccount", cloudAccount.value()); @@ -1904,7 +1907,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { response.setDouble("quota", deployment.quota().rate()); deployment.cost().ifPresent(cost -> response.setDouble("cost", cost)); - controller.archiveBucketDb().archiveUriFor(deploymentId.zoneId(), deploymentId.applicationId().tenant(), false) + (controller.zoneRegistry().isEnclave(deployment.cloudAccount()) ? + controller.archiveBucketDb().archiveUriFor(deploymentId.zoneId(), deployment.cloudAccount(), false) : + controller.archiveBucketDb().archiveUriFor(deploymentId.zoneId(), deploymentId.applicationId().tenant(), false)) .ifPresent(archiveUri -> response.setString("archiveUri", archiveUri.toString())); Cursor activity = response.setObject("activity"); @@ -1997,7 +2002,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { .forEach(endpoint -> { Cursor endpointObject = endpointsArray.addObject(); endpointObject.setString("endpointId", endpoint.endpointId()); - endpointObject.setString("state", endpoint.state()); + endpointObject.setString("state", endpoint.stateValue().name()); + endpointObject.setString("detail", endpoint.stateString()); }); }); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java index 5928a50c907..bc7dd4199c7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java @@ -222,11 +222,16 @@ public class BillingApiHandler extends ThreadedHttpRequestHandler { private HttpResponse addLineItem(HttpRequest request, String tenant, String userId) { Inspector inspector = inspectorOrThrow(request); + + Optional<Bill.Id> billId = SlimeUtils.optionalString(inspector.field("billId")).map(Bill.Id::of); + billingController.addLineItem( TenantName.from(tenant), getInspectorFieldOrThrow(inspector, "description"), new BigDecimal(getInspectorFieldOrThrow(inspector, "amount")), + billId, userId); + return new MessageResponse("Added line item for tenant " + tenant); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index 61c71e964f6..637393d71cb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -2,10 +2,8 @@ package com.yahoo.vespa.hosted.controller.routing; import ai.vespa.http.DomainName; -import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.transaction.Mutex; @@ -22,7 +20,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record.Type; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.DnsChallenge; -import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.State; +import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.ChallengeState; import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedAliasTarget; import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedDirectTarget; import com.yahoo.vespa.hosted.controller.application.Endpoint; @@ -30,13 +28,10 @@ import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder; -import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; import com.yahoo.vespa.hosted.controller.dns.NameServiceRequest; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.yolean.UncheckedInterruptedException; -import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -415,12 +410,12 @@ public class RoutingPolicies { .collect(Collectors.toSet()); try { challenges.removeIf(challenge -> { - if (challenge.state() == State.pending) { + if (challenge.state() == ChallengeState.pending) { if (pendingRequests.contains(challenge.name())) return false; - challenge = challenge.withState(State.ready); + challenge = challenge.withState(ChallengeState.ready); } - State state = controller.serviceRegistry().vpcEndpointService().process(challenge); - if (state == State.done) { + ChallengeState state = controller.serviceRegistry().vpcEndpointService().process(challenge); + if (state == ChallengeState.done) { removeDnsChallenge(challenge); return true; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java index a199ef9e34e..cb9c1c2fa13 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java @@ -6,6 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneId; @@ -64,7 +65,7 @@ public class DeploymentQuotaCalculatorTest { var existing_dev_deployment = new Application(TenantAndApplicationId.from(ApplicationId.defaultId()), Instant.EPOCH, DeploymentSpec.empty, ValidationOverrides.empty, Optional.empty(), Optional.empty(), Optional.empty(), OptionalInt.empty(), new ApplicationMetrics(1, 1), Set.of(), OptionalLong.empty(), RevisionHistory.empty(), List.of(new Instance(ApplicationId.defaultId()).withNewDeployment(ZoneId.from(Environment.dev, RegionName.defaultName()), - RevisionId.forProduction(1), Version.emptyVersion, Instant.EPOCH, Map.of(), QuotaUsage.create(0.53d)))); + RevisionId.forProduction(1), Version.emptyVersion, Instant.EPOCH, Map.of(), QuotaUsage.create(0.53d), CloudAccount.empty))); Quota calculated = DeploymentQuotaCalculator.calculate(Quota.unlimited().withBudget(2), List.of(existing_dev_deployment), ApplicationId.defaultId(), ZoneId.defaultId(), DeploymentSpec.fromXml( diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDbTest.java index 081056e5184..e5571c0e0ca 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDbTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDbTest.java @@ -1,15 +1,21 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.archive; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.test.ManualClock; +import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBuckets; +import com.yahoo.vespa.hosted.controller.api.integration.archive.MockArchiveService; +import com.yahoo.vespa.hosted.controller.api.integration.archive.VespaManagedArchiveBucket; import org.apache.curator.shaded.com.google.common.collect.Streams; import org.junit.jupiter.api.Test; import java.net.URI; +import java.time.Duration; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -21,27 +27,27 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class CuratorArchiveBucketDbTest { @Test - void archiveUriFor() { + void archiveUriForTenant() { ControllerTester tester = new ControllerTester(SystemName.Public); CuratorArchiveBucketDb bucketDb = new CuratorArchiveBucketDb(tester.controller()); tester.curator().writeArchiveBuckets(ZoneId.defaultId(), - Set.of(new ArchiveBucket("existingBucket", "keyArn").withTenant(TenantName.defaultName()))); + ArchiveBuckets.EMPTY.with(new VespaManagedArchiveBucket("existingBucket", "keyArn").withTenant(TenantName.defaultName()))); // Finds existing bucket in db - assertEquals(Optional.of(URI.create("s3://existingBucket/default/")), bucketDb.archiveUriFor(ZoneId.defaultId(), TenantName.defaultName(), true)); + assertEquals(Optional.of(URI.create("s3://existingBucket/")), bucketDb.archiveUriFor(ZoneId.defaultId(), TenantName.defaultName(), true)); // Assigns to existing bucket while there is space IntStream.range(0, 4).forEach(i -> assertEquals( - Optional.of(URI.create("s3://existingBucket/tenant" + i + "/")), bucketDb + Optional.of(URI.create("s3://existingBucket/")), bucketDb .archiveUriFor(ZoneId.defaultId(), TenantName.from("tenant" + i), true))); // Creates new bucket when existing buckets are full - assertEquals(Optional.of(URI.create("s3://bucketName/lastDrop/")), bucketDb.archiveUriFor(ZoneId.defaultId(), TenantName.from("lastDrop"), true)); + assertEquals(Optional.of(URI.create("s3://bucketName/")), bucketDb.archiveUriFor(ZoneId.defaultId(), TenantName.from("lastDrop"), true)); // Creates new bucket when there are no existing buckets in zone - assertEquals(Optional.of(URI.create("s3://bucketName/firstInZone/")), bucketDb.archiveUriFor(ZoneId.from("prod.us-east-3"), TenantName.from("firstInZone"), true)); + assertEquals(Optional.of(URI.create("s3://bucketName/")), bucketDb.archiveUriFor(ZoneId.from("prod.us-east-3"), TenantName.from("firstInZone"), true)); // Does not create bucket if not required assertEquals(Optional.empty(), bucketDb.archiveUriFor(ZoneId.from("prod.us-east-3"), TenantName.from("newTenant"), false)); @@ -50,11 +56,36 @@ public class CuratorArchiveBucketDbTest { Set<TenantName> existingBucketTenants = Streams.concat(Stream.of(TenantName.defaultName()), IntStream.range(0, 4).mapToObj(i -> TenantName.from("tenant" + i))).collect(Collectors.toUnmodifiableSet()); assertEquals( Set.of( - new ArchiveBucket("existingBucket", "keyArn").withTenants(existingBucketTenants), - new ArchiveBucket("bucketName", "keyArn").withTenant(TenantName.from("lastDrop"))), - bucketDb.buckets(ZoneId.defaultId())); + new VespaManagedArchiveBucket("existingBucket", "keyArn").withTenants(existingBucketTenants), + new VespaManagedArchiveBucket("bucketName", "keyArn").withTenant(TenantName.from("lastDrop"))), + bucketDb.buckets(ZoneId.defaultId()).vespaManaged()); assertEquals( - Set.of(new ArchiveBucket("bucketName", "keyArn").withTenant(TenantName.from("firstInZone"))), - bucketDb.buckets(ZoneId.from("prod.us-east-3"))); + Set.of(new VespaManagedArchiveBucket("bucketName", "keyArn").withTenant(TenantName.from("firstInZone"))), + bucketDb.buckets(ZoneId.from("prod.us-east-3")).vespaManaged()); + } + + @Test + void archiveUriForAccount() { + Controller controller = new ControllerTester(SystemName.Public).controller(); + CuratorArchiveBucketDb bucketDb = new CuratorArchiveBucketDb(controller); + MockArchiveService service = (MockArchiveService) controller.serviceRegistry().archiveService(); + ManualClock clock = (ManualClock) controller.clock(); + + CloudAccount acc1 = CloudAccount.from("001122334455"); + ZoneId z1 = ZoneId.from("prod.us-east-3"); + + assertEquals(Optional.empty(), bucketDb.archiveUriFor(z1, acc1, true)); // Initially not set + service.setEnclaveArchiveBucket(z1, acc1, "bucket-1"); + assertEquals(Optional.empty(), bucketDb.archiveUriFor(z1, acc1, false)); + assertEquals(Optional.of(URI.create("s3://bucket-1/")), bucketDb.archiveUriFor(z1, acc1, true)); + assertEquals(Optional.of(URI.create("s3://bucket-1/")), bucketDb.archiveUriFor(z1, acc1, false)); + + service.setEnclaveArchiveBucket(z1, acc1, "bucket-2"); + assertEquals(Optional.of(URI.create("s3://bucket-1/")), bucketDb.archiveUriFor(z1, acc1, true)); // Returns old value even with search + + clock.advance(Duration.ofMinutes(61)); // After expiry the cache is expired, new search is performed + assertEquals(Optional.of(URI.create("s3://bucket-1/")), bucketDb.archiveUriFor(z1, acc1, false)); // When requesting without search, return previous value even if expired + assertEquals(Optional.of(URI.create("s3://bucket-2/")), bucketDb.archiveUriFor(z1, acc1, true)); + assertEquals(Optional.of(URI.create("s3://bucket-2/")), bucketDb.archiveUriFor(z1, acc1, false)); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 9895cd68004..226fb785bf6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -8,7 +8,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 37ef85b991b..297997365b0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.integration; import com.yahoo.collections.Pair; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; @@ -11,8 +12,10 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveUriUpdate; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Application; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationStats; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ArchiveUris; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Load; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; @@ -22,6 +25,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.TargetVers import com.yahoo.vespa.hosted.controller.api.integration.noderepository.ApplicationPatch; import java.net.URI; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -41,7 +45,7 @@ public class NodeRepositoryMock implements NodeRepository { private final Map<ZoneId, TargetVersions> targetVersions = new ConcurrentHashMap<>(); private final Map<DeploymentId, Pair<Double, Double>> trafficFractions = new ConcurrentHashMap<>(); private final Map<DeploymentClusterId, BcpGroupInfo> bcpGroupInfos = new ConcurrentHashMap<>(); - private final Map<ZoneId, Map<TenantName, URI>> archiveUris = new ConcurrentHashMap<>(); + private final Map<ZoneId, ArchiveUris> archiveUris = new ConcurrentHashMap<>(); private boolean allowPatching = true; private boolean hasSpareCapacity = false; @@ -118,18 +122,26 @@ public class NodeRepositoryMock implements NodeRepository { } @Override - public Map<TenantName, URI> getArchiveUris(ZoneId zone) { - return Map.copyOf(archiveUris.getOrDefault(zone, Map.of())); + public ArchiveUris getArchiveUris(ZoneId zone) { + return archiveUris.getOrDefault(zone, ArchiveUris.EMPTY); } @Override - public void setArchiveUri(ZoneId zone, TenantName tenantName, URI archiveUri) { - archiveUris.computeIfAbsent(zone, z -> new ConcurrentHashMap<>()).put(tenantName, archiveUri); - } - - @Override - public void removeArchiveUri(ZoneId zone, TenantName tenantName) { - Optional.ofNullable(archiveUris.get(zone)).ifPresent(map -> map.remove(tenantName)); + public void updateArchiveUri(ZoneId zone, ArchiveUriUpdate update) { + archiveUris.compute(zone, (z, prev) -> { + prev = prev == null ? ArchiveUris.EMPTY : prev; + if (update.tenantName().isPresent()) { + Map<TenantName, URI> updated = new HashMap<>(prev.tenantArchiveUris()); + update.archiveUri().ifPresentOrElse(uri -> updated.put(update.tenantName().get(), uri), + () -> updated.remove(update.tenantName().get())); + return new ArchiveUris(updated, prev.accountArchiveUris()); + } else { + Map<CloudAccount, URI> updated = new HashMap<>(prev.accountArchiveUris()); + update.archiveUri().ifPresentOrElse(uri -> updated.put(update.cloudAccount().get(), uri), + () -> updated.remove(update.cloudAccount().get())); + return new ArchiveUris(prev.tenantArchiveUris(), updated); + } + }); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index be257daa211..998b371bbf1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -37,11 +37,11 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueH import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumerMock; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClientMock; +import com.yahoo.vespa.hosted.controller.api.integration.secrets.EndpointSecretManager; import com.yahoo.vespa.hosted.controller.api.integration.secrets.GcpSecretStore; -import com.yahoo.vespa.hosted.controller.api.integration.secrets.NoopGcpSecretStore; import com.yahoo.vespa.hosted.controller.api.integration.secrets.NoopEndpointSecretManager; +import com.yahoo.vespa.hosted.controller.api.integration.secrets.NoopGcpSecretStore; import com.yahoo.vespa.hosted.controller.api.integration.secrets.NoopTenantSecretService; -import com.yahoo.vespa.hosted.controller.api.integration.secrets.EndpointSecretManager; import com.yahoo.vespa.hosted.controller.api.integration.stubs.DummyOwnershipIssues; import com.yahoo.vespa.hosted.controller.api.integration.stubs.DummySystemMonitor; import com.yahoo.vespa.hosted.controller.api.integration.stubs.LoggingDeploymentIssues; @@ -89,7 +89,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final ArtifactRegistryMock containerRegistry = new ArtifactRegistryMock(); private final NoopTenantSecretService tenantSecretService = new NoopTenantSecretService(); private final NoopEndpointSecretManager secretManager = new NoopEndpointSecretManager(); - private final ArchiveService archiveService = new MockArchiveService(); + private final ArchiveService archiveService = new MockArchiveService(clock); private final MockChangeRequestClient changeRequestClient = new MockChangeRequestClient(); private final AccessControlService accessControlService = new MockAccessControlService(); private final HorizonClient horizonClient = new MockHorizonClient(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 38ff9967ef6..e59c677d0fa 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -50,6 +50,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry private final SystemName system; // Don't even think about making it non-final! ƪ(`▿▿▿▿´ƪ) private List<? extends ZoneApi> zones; + private CloudAccount systemCloudAccount = CloudAccount.from("111333555777"); private UpgradePolicy upgradePolicy = null; /** @@ -266,7 +267,12 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry @Override public boolean hasZone(ZoneId zoneId, CloudAccount cloudAccount) { - return hasZone(zoneId) && cloudAccountZones.getOrDefault(cloudAccount, Set.of()).contains(zoneId); + return hasZone(zoneId) && (system.isPublic() || cloudAccountZones.getOrDefault(cloudAccount, Set.of()).contains(zoneId)); + } + + @Override + public boolean isEnclave(CloudAccount cloudAccount) { + return system.isPublic() && !cloudAccount.isUnspecified() && !cloudAccount.equals(systemCloudAccount); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java index c10b77d853a..0490a9bdcc5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.jdisc.test.MockMetric; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.LockedTenant; -import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; import com.yahoo.vespa.hosted.controller.api.integration.archive.MockArchiveService; import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.Tenant; @@ -15,8 +14,6 @@ import org.junit.jupiter.api.Test; import java.time.Duration; import java.util.Map; -import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -37,7 +34,6 @@ public class ArchiveAccessMaintainerTest { ZoneId testZone = ZoneId.from("prod.aws-us-east-1c"); tester.controller().archiveBucketDb().archiveUriFor(testZone, tenant1, true); - var testBucket = new ArchiveBucket("bucketName", "keyArn").withTenant(tenant1); MockArchiveService archiveService = (MockArchiveService) tester.controller().serviceRegistry().archiveService(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java index 8c44b39691c..14540971faf 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java @@ -1,23 +1,28 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.component.Version; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBuckets; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveUriUpdate; +import com.yahoo.vespa.hosted.controller.api.integration.archive.MockArchiveService; +import com.yahoo.vespa.hosted.controller.api.integration.archive.VespaManagedArchiveBucket; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ArchiveUris; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import org.junit.jupiter.api.Test; import java.net.URI; import java.time.Duration; -import java.util.LinkedHashSet; +import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -36,51 +41,62 @@ public class ArchiveUriUpdaterTest { var tenant1 = TenantName.from("tenant1"); var tenant2 = TenantName.from("tenant2"); + var account1 = CloudAccount.from("001122334455"); var tenantInfra = SystemApplication.TENANT; - var application = tester.newDeploymentContext(tenant1.value(), "app1", "instance1"); ZoneId zone = ZoneId.from("prod", "aws-us-east-1c"); // Initially we should only is the bucket for hosted-vespa tenant updater.maintain(); - assertArchiveUris(Map.of(TenantName.from("hosted-vespa"), "s3://bucketName/hosted-vespa/"), zone); - assertArchiveUris(Map.of(TenantName.from("hosted-vespa"), "s3://bucketName/hosted-vespa/"), ZoneId.from("prod", "controller")); + assertArchiveUris(zone, Map.of(TenantName.from("hosted-vespa"), "s3://bucketName/"), Map.of()); + assertArchiveUris(ZoneId.from("prod", "controller"), Map.of(TenantName.from("hosted-vespa"), "s3://bucketName/"), Map.of()); // Archive service now has URI for tenant1, but tenant1 is not deployed in zone - setBucketNameInService(Map.of(tenant1, "uri-1"), zone); + setBucketNameInService(Map.of(tenant2, "uri-1"), zone); + setAccountBucketNameInService(zone, account1, "bkt-1"); updater.maintain(); - assertArchiveUris(Map.of(TenantName.from("hosted-vespa"), "s3://bucketName/hosted-vespa/"), zone); + assertArchiveUris(zone, Map.of(TenantName.from("hosted-vespa"), "s3://bucketName/"), Map.of()); + + ((InMemoryFlagSource) tester.controller().flagSource()) + .withListFlag(PermanentFlags.CLOUD_ACCOUNTS.id(), List.of(account1.value()), String.class); + deploy(tester.newDeploymentContext(tenant1.value(), "app1", "instance1"), zone, account1); + deploy(tester.newDeploymentContext(tenant2.value(), "app1", "instance1"), zone, CloudAccount.empty); - deploy(application, zone); updater.maintain(); - assertArchiveUris(Map.of(tenant1, "s3://uri-1/tenant1/", tenantInfra, "s3://bucketName/hosted-vespa/"), zone); + assertArchiveUris(zone, Map.of(tenant2, "s3://uri-1/", tenantInfra, "s3://bucketName/"), Map.of(account1, "s3://bkt-1/")); // URI for tenant1 should be updated and removed for tenant2 setArchiveUriInNodeRepo(Map.of(tenant1, "wrong-uri", tenant2, "uri-2"), zone); updater.maintain(); - assertArchiveUris(Map.of(tenant1, "s3://uri-1/tenant1/", tenantInfra, "s3://bucketName/hosted-vespa/"), zone); + assertArchiveUris(zone, Map.of(tenant2, "s3://uri-1/", tenantInfra, "s3://bucketName/"), Map.of(account1, "s3://bkt-1/")); } - private void assertArchiveUris(Map<TenantName, String> expectedUris, ZoneId zone) { - Map<TenantName, String> actualUris = tester.controller().serviceRegistry().configServer().nodeRepository() - .getArchiveUris(zone).entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString())); - assertEquals(expectedUris, actualUris); + private void assertArchiveUris(ZoneId zone, Map<TenantName, String> expectedTenantUris, Map<CloudAccount, String> expectedAccountUris) { + ArchiveUris archiveUris = tester.controller().serviceRegistry().configServer().nodeRepository().getArchiveUris(zone); + assertEquals(expectedTenantUris, archiveUris.tenantArchiveUris().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString()))); + assertEquals(expectedAccountUris, archiveUris.accountArchiveUris().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString()))); } private void setBucketNameInService(Map<TenantName, String> bucketNames, ZoneId zone) { - var archiveBuckets = new LinkedHashSet<>(tester.controller().curator().readArchiveBuckets(zone)); - bucketNames.forEach((tenantName, bucketName) -> - archiveBuckets.add(new ArchiveBucket(bucketName, "keyArn").withTenant(tenantName))); - tester.controller().curator().writeArchiveBuckets(zone, archiveBuckets); + ArchiveBuckets buckets = tester.controller().curator().readArchiveBuckets(zone); + for (var entry : bucketNames.entrySet()) + buckets = buckets.with(new VespaManagedArchiveBucket(entry.getValue(), "keyArn").withTenant(entry.getKey())); + tester.controller().curator().writeArchiveBuckets(zone, buckets); + } + + private void setAccountBucketNameInService(ZoneId zone, CloudAccount cloudAccount, String bucketName) { + ((MockArchiveService) tester.controller().serviceRegistry().archiveService()).setEnclaveArchiveBucket(zone, cloudAccount, bucketName); } private void setArchiveUriInNodeRepo(Map<TenantName, String> archiveUris, ZoneId zone) { NodeRepository nodeRepository = tester.controller().serviceRegistry().configServer().nodeRepository(); - archiveUris.forEach((tenant, uri) -> nodeRepository.setArchiveUri(zone, tenant, URI.create(uri))); + archiveUris.forEach((tenant, uri) -> nodeRepository.updateArchiveUri(zone, ArchiveUriUpdate.setArchiveUriFor(tenant, URI.create(uri)))); } - private void deploy(DeploymentContext application, ZoneId zone) { - application.runJob(JobType.deploymentTo(zone), new ApplicationPackage(new byte[0])); + private void deploy(DeploymentContext application, ZoneId zone, CloudAccount cloudAccount) { + application.submit(new ApplicationPackageBuilder() + .cloudAccount(cloudAccount.value()) + .region(zone.region().value()) + .build()).deploy(); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java index 47a1b44d196..934e15ad623 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; @@ -143,7 +144,7 @@ public class EndpointCertificateMaintainerTest { @NotNull private EndpointCertificateMaintainer.EligibleJob makeDeploymentAtAge(int ageInDays) { - var deployment = new Deployment(ZoneId.defaultId(), RevisionId.forProduction(1), Version.emptyVersion, + var deployment = new Deployment(ZoneId.defaultId(), CloudAccount.empty, RevisionId.forProduction(1), Version.emptyVersion, Instant.now().minus(ageInDays, ChronoUnit.DAYS), DeploymentMetrics.none, DeploymentActivity.none, QuotaUsage.none, OptionalDouble.empty()); return new EndpointCertificateMaintainer.EligibleJob(deployment, ApplicationId.defaultId(), JobType.prod("somewhere")); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java index 834777abb62..04da0105e99 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java @@ -12,6 +12,7 @@ import com.yahoo.test.ManualClock; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing; @@ -79,7 +80,7 @@ public class NotificationsDbTest { private final ManualClock clock = new ManualClock(Instant.ofEpochSecond(12345)); private final MockCuratorDb curatorDb = new MockCuratorDb(SystemName.Public); private final MockMailer mailer = new MockMailer(); - private final FlagSource flagSource = new InMemoryFlagSource().withBooleanFlag(Flags.NOTIFICATION_DISPATCH_FLAG.id(), true); + private final FlagSource flagSource = new InMemoryFlagSource().withBooleanFlag(PermanentFlags.NOTIFICATION_DISPATCH_FLAG.id(), true); private final NotificationsDb notificationsDb = new NotificationsDb(clock, curatorDb, new Notifier(curatorDb, new ZoneRegistryMock(SystemName.cd), mailer, flagSource)); @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java index 0c031a13e6f..96edba27c6f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; @@ -58,7 +59,7 @@ public class NotifierTest { @Test void dispatch() throws IOException { var mailer = new MockMailer(); - var flagSource = new InMemoryFlagSource().withBooleanFlag(Flags.NOTIFICATION_DISPATCH_FLAG.id(), true); + var flagSource = new InMemoryFlagSource().withBooleanFlag(PermanentFlags.NOTIFICATION_DISPATCH_FLAG.id(), true); var notifier = new Notifier(curatorDb, new ZoneRegistryMock(SystemName.cd), mailer, flagSource); var notification = new Notification(Instant.now(), Notification.Type.testPackage, Notification.Level.warning, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index faef6de94ca..589fc25700f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneId; @@ -111,9 +112,9 @@ public class ApplicationSerializerTest { Version.fromString("6.3.1"), Instant.ofEpochMilli(496)); Instant activityAt = Instant.parse("2018-06-01T10:15:30.00Z"); - deployments.add(new Deployment(zone1, applicationVersion1.id(), Version.fromString("1.2.3"), Instant.ofEpochMilli(3), + deployments.add(new Deployment(zone1, CloudAccount.empty, applicationVersion1.id(), Version.fromString("1.2.3"), Instant.ofEpochMilli(3), DeploymentMetrics.none, DeploymentActivity.none, QuotaUsage.none, OptionalDouble.empty())); - deployments.add(new Deployment(zone2, applicationVersion2.id(), Version.fromString("1.2.3"), Instant.ofEpochMilli(5), + deployments.add(new Deployment(zone2, CloudAccount.from("001122334455"), applicationVersion2.id(), Version.fromString("1.2.3"), Instant.ofEpochMilli(5), new DeploymentMetrics(2, 3, 4, 5, 6, Optional.of(Instant.now().truncatedTo(ChronoUnit.MILLIS)), Map.of(DeploymentMetrics.Warning.all, 3)), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializerTest.java index 82c5a6fc0c1..1d1b1124d22 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializerTest.java @@ -1,11 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBuckets; +import com.yahoo.vespa.hosted.controller.api.integration.archive.TenantManagedArchiveBucket; +import com.yahoo.vespa.hosted.controller.api.integration.archive.VespaManagedArchiveBucket; import org.junit.jupiter.api.Test; -import java.util.LinkedHashSet; +import java.time.Instant; +import java.util.Set; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -13,17 +17,12 @@ public class ArchiveBucketsSerializerTest { @Test void serdes() { - var testTenants = new LinkedHashSet<TenantName>(); - testTenants.add(TenantName.from("tenant1")); - testTenants.add(TenantName.from("tenant2")); + ArchiveBuckets archiveBuckets = new ArchiveBuckets( + Set.of(new VespaManagedArchiveBucket("bucket1Name", "key1Arn").withTenants(Set.of(TenantName.from("t1"), TenantName.from("t2"))), + new VespaManagedArchiveBucket("bucket2Name", "key2Arn").withTenant(TenantName.from("t3"))), + Set.of(new TenantManagedArchiveBucket("bucket3Name", CloudAccount.from("acct-1"), Instant.ofEpochMilli(1234)), + new TenantManagedArchiveBucket("bucket4Name", CloudAccount.from("acct-2"), Instant.ofEpochMilli(5678)))); - var testBuckets = new LinkedHashSet<ArchiveBucket>(); - testBuckets.add(new ArchiveBucket("bucket1Name", "key1Arn").withTenants(testTenants)); - testBuckets.add(new ArchiveBucket("bucket2Name", "key2Arn")); - - String zkData = "{\"buckets\":[{\"bucketName\":\"bucket1Name\",\"keyArn\":\"key1Arn\",\"tenantIds\":[\"tenant1\",\"tenant2\"]},{\"bucketName\":\"bucket2Name\",\"keyArn\":\"key2Arn\",\"tenantIds\":[]}]}"; - - assertEquals(testBuckets, ArchiveBucketsSerializer.fromJsonString(zkData)); - assertEquals(testBuckets, ArchiveBucketsSerializer.fromJsonString(ArchiveBucketsSerializer.toSlime(testBuckets).toString())); + assertEquals(archiveBuckets, ArchiveBucketsSerializer.fromSlime(ArchiveBucketsSerializer.toSlime(archiveBuckets))); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/DnsChallengeSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/DnsChallengeSerializerTest.java index 674cc29e91c..b224975fe05 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/DnsChallengeSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/DnsChallengeSerializerTest.java @@ -9,7 +9,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.DnsChallenge; -import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.State; +import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.ChallengeState; import org.junit.jupiter.api.Test; import java.time.Instant; @@ -32,13 +32,13 @@ class DnsChallengeSerializerTest { "deadbeef", Optional.of(CloudAccount.from("123321123321")), Instant.ofEpochMilli(123), - State.pending); + ChallengeState.pending); @Test void testSerialization() { DnsChallenge deserialized = serializer.fromJson(serializer.toJson(challenge), clusterId); assertEquals(challenge, deserialized); - for (State state : State.values()) + for (ChallengeState state : ChallengeState.values()) assertEquals(challenge.withState(state), serializer.fromJson(serializer.toJson(challenge.withState(state)), clusterId)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index 8a37bb560e2..6012b491fe7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -77,7 +77,15 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { .roles(Set.of(Role.administrator(tenantName))); tester.assertResponse(updateRequest, "{\"message\":\"Tenant info updated\"}", 200); - tester.assertResponse(request, "{\"contact\":{\"name\":\"Some Name\",\"email\":\"foo@example.com\",\"emailVerified\":false},\"tenant\":{\"company\":\"\",\"website\":\"https://example.com/\"}}", 200); + tester.assertResponse(request, "{\"contact\":{\"name\":\"Some Name\",\"email\":\"foo@example.com\",\"emailVerified\":false},\"tenant\":{\"company\":\"Scoober, Inc.\",\"website\":\"https://example.com/\"}}", 200); + } + + @Test + void tenant_info_profile_too_long() { + var request = request("/application/v4/tenant/scoober/info/profile", PUT) + .data("{\"contact\":{\"name\":\"" + "a".repeat(513) + "\",\"email\":\"foo@example.com\"},\"tenant\":{\"company\":\"Scoober, Inc.\",\"website\":\"https://example.com/\"}}") + .roles(Set.of(Role.administrator(tenantName))); + tester.assertResponse(request, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Input value too long\"}", 400); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index f1b061da58b..14771906852 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -701,7 +701,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/private-services", GET) .userIdentity(USER_ID), """ - {"privateServices":[{"cluster":"default","serviceId":"service","type":"unknown","allowedUrns":[{"type":"aws-private-link","urn":"arne"}],"endpoints":[{"endpointId":"endpoint-1","state":"available"}]}]}"""); + {"privateServices":[{"cluster":"default","serviceId":"service","type":"unknown","allowedUrns":[{"type":"aws-private-link","urn":"arne"}],"endpoints":[{"endpointId":"endpoint-1","state":"open","detail":"available"}]}]}"""); // GET service/state/v1 tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/service/storagenode/host.com/state/v1/?foo=bar", GET) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java index 79007a4439a..f247b0ed3b6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java @@ -22,6 +22,7 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.TreeMap; @@ -170,7 +171,7 @@ public class BillingApiHandlerTest extends ControllerContainerCloudTest { billingController.setPlan(tenant, PlanId.from("some-plan"), true, false); billingController.setPlan(tenant2, PlanId.from("some-plan"), true, false); billingController.addBill(tenant, bill, false); - billingController.addLineItem(tenant, "support", new BigDecimal("42"), "Smith"); + billingController.addLineItem(tenant, "support", new BigDecimal("42"), Optional.empty(), "Smith"); billingController.addBill(tenant2, bill, false); var request = request("/billing/v1/billing?until=2020-05-28").roles(financeAdmin); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 94cffb94184..a90d942bd09 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -9,7 +9,6 @@ import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; -import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; @@ -25,8 +24,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record.Type; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.DnsChallenge; -import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.State; +import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.ChallengeState; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.EndpointList; @@ -52,8 +50,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -537,7 +533,7 @@ public class RoutingPoliciesTest { tester.tester.controllerTester().serviceRegistry().vpcEndpointService().outcomes - .put(RecordName.from("challenge--a.t.aws-us-east-1a.vespa.oath.cloud"), State.running); + .put(RecordName.from("challenge--a.t.aws-us-east-1a.vespa.oath.cloud"), ChallengeState.running); // Deployment fails because challenge is not answered (immediately). assertEquals("Status of run 2 of production-aws-us-east-1a for t.a ==> expected: <succeeded> but was: <unfinished>", diff --git a/dist/vespa.spec b/dist/vespa.spec index 56b849a8135..5ddeab3b63c 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -54,8 +54,7 @@ BuildRequires: gcc-toolset-12-libatomic-devel BuildRequires: maven BuildRequires: maven-openjdk17 BuildRequires: vespa-pybind11-devel -BuildRequires: python3-pytest -BuildRequires: python36-devel +BuildRequires: python39-devel BuildRequires: glibc-langpack-en %endif %if 0%{?el9} @@ -94,8 +93,9 @@ BuildRequires: vespa-gtest = 1.11.0 %define _use_vespa_gtest 1 BuildRequires: vespa-lz4-devel >= 1.9.4-1 BuildRequires: vespa-onnxruntime-devel = 1.13.1 -BuildRequires: vespa-protobuf-devel = 3.21.7 -BuildRequires: vespa-libzstd-devel >= 1.5.2-1 +BuildRequires: vespa-protobuf-devel = 3.21.12 +%define _use_vespa_protobuf 1 +BuildRequires: vespa-libzstd-devel >= 1.5.4-1 %endif %if 0%{?el9} BuildRequires: cmake >= 3.20.2 @@ -104,8 +104,9 @@ BuildRequires: maven-openjdk17 BuildRequires: openssl-devel BuildRequires: vespa-lz4-devel >= 1.9.4-1 BuildRequires: vespa-onnxruntime-devel = 1.13.1 -BuildRequires: vespa-libzstd-devel >= 1.5.2-1 -BuildRequires: vespa-protobuf-devel = 3.21.7 +BuildRequires: vespa-libzstd-devel >= 1.5.4-1 +BuildRequires: vespa-protobuf-devel = 3.21.12 +%define _use_vespa_protobuf 1 BuildRequires: llvm-devel BuildRequires: boost-devel >= 1.75 BuildRequires: gtest-devel @@ -125,7 +126,7 @@ BuildRequires: maven-openjdk17 BuildRequires: openssl-devel BuildRequires: vespa-lz4-devel >= 1.9.4-1 BuildRequires: vespa-onnxruntime-devel = 1.13.1 -BuildRequires: vespa-libzstd-devel >= 1.5.2-1 +BuildRequires: vespa-libzstd-devel >= 1.5.4-1 BuildRequires: protobuf-devel BuildRequires: llvm-devel BuildRequires: boost-devel @@ -210,7 +211,7 @@ Requires: %{name}-tools = %{version}-%{release} # Ugly workaround because vespamalloc/src/vespamalloc/malloc/mmap.cpp uses the private # _dl_sym function. # Exclude automated requires for libraries in /opt/vespa-deps/lib64. -%global __requires_exclude ^lib(c\\.so\\.6\\(GLIBC_PRIVATE\\)|pthread\\.so\\.0\\(GLIBC_PRIVATE\\)|(icui18n|icuuc|lz4|protobuf|zstd|onnxruntime%{?_use_vespa_openssl:|crypto|ssl}%{?_use_vespa_openblas:|openblas}%{?_use_vespa_re2:|re2}%{?_use_vespa_xxhash:|xxhash}%{?_use_vespa_gtest:|(gtest|gmock)(_main)?})\\.so\\.[0-9.]*\\([A-Za-z._0-9]*\\))\\(64bit\\)$ +%global __requires_exclude ^lib(c\\.so\\.6\\(GLIBC_PRIVATE\\)|pthread\\.so\\.0\\(GLIBC_PRIVATE\\)|(lz4%{?_use_vespa_protobuf:|protobuf}|zstd|onnxruntime%{?_use_vespa_openssl:|crypto|ssl}%{?_use_vespa_openblas:|openblas}%{?_use_vespa_re2:|re2}%{?_use_vespa_xxhash:|xxhash}%{?_use_vespa_gtest:|(gtest|gmock)(_main)?})\\.so\\.[0-9.]*\\([A-Za-z._0-9]*\\))\\(64bit\\)$ %description @@ -253,7 +254,7 @@ Requires: vespa-openssl >= 1.1.1o-1 Requires: openssl-libs %endif Requires: vespa-lz4 >= 1.9.4-1 -Requires: vespa-libzstd >= 1.5.2-1 +Requires: vespa-libzstd >= 1.5.4-1 %if 0%{?el8} Requires: vespa-openblas = 0.3.21 %else @@ -285,11 +286,11 @@ Requires: openssl-libs %endif %if 0%{?el8} Requires: llvm-libs -Requires: vespa-protobuf = 3.21.7 +Requires: vespa-protobuf = 3.21.12 %endif %if 0%{?el9} Requires: llvm-libs -Requires: vespa-protobuf = 3.21.7 +Requires: vespa-protobuf = 3.21.12 %endif %if 0%{?fedora} Requires: protobuf @@ -375,7 +376,7 @@ Summary: Vespa - The open big data serving engine - ann-benchmark Requires: %{name}-base-libs = %{version}-%{release} Requires: %{name}-libs = %{version}-%{release} %if 0%{?el8} -Requires: python36 +Requires: python39 %endif %if 0%{?el9} Requires: python3 @@ -451,17 +452,34 @@ export FACTORY_VESPA_VERSION=%{version} mvn --batch-mode -e -N io.takari:maven:wrapper -Dmaven=3.6.3 %endif %{?_use_mvn_wrapper:env VESPA_MAVEN_COMMAND=$(pwd)/mvnw }sh bootstrap.sh java -%{?_use_mvn_wrapper:./mvnw}%{!?_use_mvn_wrapper:mvn} --batch-mode -nsu -T 1C install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true +%{?_use_mvn_wrapper:./mvnw}%{!?_use_mvn_wrapper:mvn} --batch-mode -nsu -T 1C install -DskipTests -Dmaven.javadoc.skip=true %{_command_cmake} -DCMAKE_INSTALL_PREFIX=%{_prefix} \ -DJAVA_HOME=$JAVA_HOME \ -DVESPA_USER=%{_vespa_user} \ -DVESPA_UNPRIVILEGED=no \ + %{_cmake_extra_opts} \ . make %{_smp_mflags} VERSION=%{version} CI=true make -C client/go install-all %endif +%check +%if ! 0%{?installdir:1} +%if 0%{?_java_home:1} +export JAVA_HOME=%{?_java_home} +%else +export JAVA_HOME=/usr/lib/jvm/java-17-openjdk +%endif +export PATH="$JAVA_HOME/bin:$PATH" +%if 0%{?el8} +python3.9 -m pip install --user pytest +%endif +export PYTHONPATH="$PYTHONPATH:/usr/local/lib/$(basename $(readlink -f $(which python3)))/site-packages" +#%{?_use_mvn_wrapper:./mvnw}%{!?_use_mvn_wrapper:mvn} --batch-mode -nsu -T 1C -Dmaven.javadoc.skip=true test +make test ARGS="--output-on-failure %{_smp_mflags}" +%endif + %install rm -rf %{buildroot} @@ -585,7 +603,6 @@ fi %{_prefix}/include %dir %{_prefix}/lib %dir %{_prefix}/lib/jars -%{_prefix}/lib/jars/athenz-identity-provider-service-jar-with-dependencies.jar %{_prefix}/lib/jars/cloud-tenant-cd-jar-with-dependencies.jar %{_prefix}/lib/jars/clustercontroller-apps-jar-with-dependencies.jar %{_prefix}/lib/jars/clustercontroller-core-jar-with-dependencies.jar @@ -688,7 +705,6 @@ fi %endif %dir %{_prefix} %dir %{_prefix}/lib64 -%{_prefix}/lib64/libfastos.so %{_prefix}/lib64/libfnet.so %{_prefix}/lib64/libvespadefaults.so %{_prefix}/lib64/libvespalib.so @@ -700,7 +716,6 @@ fi %endif %dir %{_prefix} %{_prefix}/lib64 -%exclude %{_prefix}/lib64/libfastos.so %exclude %{_prefix}/lib64/libfnet.so %exclude %{_prefix}/lib64/libvespadefaults.so %exclude %{_prefix}/lib64/libvespalib.so diff --git a/document/CMakeLists.txt b/document/CMakeLists.txt index 88dbe2816d9..e1e4d8ff5cc 100644 --- a/document/CMakeLists.txt +++ b/document/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib config_cloudconfig diff --git a/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java index 199ca199667..dc5cf609381 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java @@ -94,11 +94,10 @@ public class BoolFieldValue extends FieldValue { @Override public boolean equals(Object o) { if (this == o) return true; - if ( ! (o instanceof BoolFieldValue)) return false; + if ( ! (o instanceof BoolFieldValue other)) return false; if ( ! super.equals(o)) return false; - BoolFieldValue that = (BoolFieldValue) o; - return (value == that.value); + return (value == other.value); } @Override diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java index b7a3589a4d0..4c4d9c78c8e 100644 --- a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java +++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java @@ -775,7 +775,7 @@ public class DocumentSelectorTestCase { @Test public void testInheritance() throws ParseException { - var s=new DocumentSelector("parent.parentField = \"parentValue\""); + new DocumentSelector("parent.parentField = \"parentValue\""); List<DocumentPut> documents = createDocs(); assertEquals(Result.TRUE, evaluate("test", documents.get(0))); assertEquals("Matching on type is exact", Result.FALSE, evaluate("parent", documents.get(0))); diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp index ce4b69419a3..5e14a080f27 100644 --- a/document/src/tests/documentselectparsertest.cpp +++ b/document/src/tests/documentselectparsertest.cpp @@ -673,8 +673,9 @@ TEST_F(DocumentSelectParserTest, operators_1) // Inherited doctypes PARSE("testdoctype2", *_doc[4], True); PARSE("testdoctype2", *_doc[3], False); - PARSE("testdoctype1", *_doc[4], False); // testdoctype2 inherits testdoctype1, but we use exact matching for types - PARSE("testdoctype1.headerval = 10", *_doc[4], Invalid); + PARSE("testdoctype1", *_doc[4], False); // testdoctype2 inherits testdoctype1, but we use exact matching for "standalone" doctype matches. + PARSE("testdoctype1.headerval = 10", *_doc[4], True); // But _field lookups_ use is-a type matching semantics. + PARSE("testdoctype2.headerval = 10", *_doc[4], True); // Exact type match with parent field also works transparently } TEST_F(DocumentSelectParserTest, operators_2) diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index 40f398ee93e..7a5d88d1013 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -1,14 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/document/test/fieldvalue_helpers.h> -#include <vespa/document/base/testdocman.h> +#include <vespa/document/annotation/spanlist.h> #include <vespa/document/base/exceptions.h> -#include <vespa/document/datatype/tensor_data_type.h> +#include <vespa/document/base/testdocman.h> #include <vespa/document/datatype/documenttype.h> +#include <vespa/document/datatype/tensor_data_type.h> #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/document/fieldvalue/tensorfieldvalue.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/repo/fixedtyperepo.h> +#include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/document/serialization/vespadocumentserializer.h> #include <vespa/document/update/addvalueupdate.h> #include <vespa/document/update/arithmeticvalueupdate.h> @@ -26,8 +29,8 @@ #include <vespa/document/util/bytebuffer.h> #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/eval/value.h> #include <vespa/eval/eval/test/value_compare.h> +#include <vespa/eval/eval/value.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exception.h> #include <vespa/vespalib/util/exceptions.h> @@ -1329,4 +1332,68 @@ TEST(DocumentUpdateTest, array_element_update_for_invalid_index_is_ignored) EXPECT_EQ(array_value, *result_array); } +struct UpdateToEmptyDocumentFixture { + std::unique_ptr<DocumentTypeRepo> repo; + const DocumentType& doc_type; + FixedTypeRepo fixed_repo; + + UpdateToEmptyDocumentFixture() + : repo(make_repo()), + doc_type(*repo->getDocumentType("test")), + fixed_repo(*repo, doc_type) + { + } + + std::unique_ptr<DocumentTypeRepo> make_repo() { + config_builder::DocumenttypesConfigBuilderHelper builder; + builder.document(222, "test", + Struct("test.header").addField("text", DataType::T_STRING), + Struct("test.body")); + return std::make_unique<DocumentTypeRepo>(builder.config()); + } + + Document::UP make_empty_doc() { + vespalib::nbostream stream; + { + Document doc(doc_type, DocumentId("id:test:test::0")); + VespaDocumentSerializer serializer(stream); + serializer.write(doc); + } + // This simulates that the document is read from e.g. the document store + return std::make_unique<Document>(*repo, stream); + } + + DocumentUpdate::UP make_update() { + auto text = std::make_unique<StringFieldValue>("hello world"); + auto span_list_up = std::make_unique<SpanList>(); + auto span_list = span_list_up.get(); + auto tree = std::make_unique<SpanTree>("my_span_tree", std::move(span_list_up)); + tree->annotate(span_list->add(std::make_unique<Span>(0, 5)), *AnnotationType::TERM); + tree->annotate(span_list->add(std::make_unique<Span>(6, 3)), *AnnotationType::TERM); + StringFieldValue::SpanTrees trees; + trees.push_back(std::move(tree)); + text->setSpanTrees(trees, fixed_repo); + + auto result = std::make_unique<DocumentUpdate>(*repo, doc_type, DocumentId("id:test:test::0")); + result->addUpdate(FieldUpdate(doc_type.getField("text")) + .addUpdate(std::make_unique<AssignValueUpdate>(std::move(text)))); + return result; + } +}; + +TEST(DocumentUpdateTest, string_field_annotations_can_be_deserialized_after_assign_update_to_empty_document) +{ + UpdateToEmptyDocumentFixture f; + auto doc = f.make_empty_doc(); + auto update = f.make_update(); + update->applyTo(*doc); + auto fv = doc->getValue("text"); + auto& text = dynamic_cast<StringFieldValue&>(*fv); + // This uses both the DocumentTypeRepo and DocumentType in order to deserialize the annotations. + auto tree = text.getSpanTrees(); + EXPECT_EQ("hello world", text.getValue()); + ASSERT_EQ(1, tree.size()); + ASSERT_EQ(2, tree[0]->numAnnotations()); +} + } // namespace document diff --git a/document/src/vespa/document/select/valuenodes.cpp b/document/src/vespa/document/select/valuenodes.cpp index 8102a944ff0..b3052cc07e2 100644 --- a/document/src/vespa/document/select/valuenodes.cpp +++ b/document/src/vespa/document/select/valuenodes.cpp @@ -20,10 +20,20 @@ LOG_SETUP(".document.select.valuenode"); namespace document::select { namespace { - bool documentTypeEqualsName(const DocumentType& type, vespalib::stringref name) - { - return (type.getName() == name); + +[[nodiscard]] bool document_type_is_a(const DocumentType& type, vespalib::stringref name) { + if (type.getName() == name) { + return true; + } + // No exact match; try to recursively match name against any types inherited from. + for (const auto* parent : type.getInheritedTypes()) { + if (document_type_is_a(*parent, name)) { + return true; + } } + return false; +} + } InvalidValueNode::InvalidValueNode(vespalib::stringref name) @@ -409,7 +419,7 @@ FieldValueNode::getValue(const Context& context) const const Document& doc = *context._doc; - if (!documentTypeEqualsName(doc.getType(), _doctype)) { + if (!document_type_is_a(doc.getType(), _doctype)) { return std::make_unique<InvalidValue>(); } // Imported fields can only be meaningfully evaluated inside Proton, so we @@ -473,7 +483,7 @@ FieldValueNode::traceValue(const Context &context, std::ostream& out) const return defaultTrace(getValue(context), out); } const Document &doc(*context._doc); - if (!documentTypeEqualsName(doc.getType(), _doctype)) { + if (!document_type_is_a(doc.getType(), _doctype)) { out << "Document is of type " << doc.getType() << " which isn't a " << _doctype << " document, thus resolving invalid.\n"; return std::make_unique<InvalidValue>(); diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp index bbe4f5373cb..8b75c8758ee 100644 --- a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp @@ -86,6 +86,7 @@ VespaDocumentDeserializer::readDocument(Document &value) { value.getFields().reset(); } value.setRepo(_repo.getDocumentTypeRepo()); + value.getFields().setDocumentType(value.getType()); FixedTypeRepo repo(_repo.getDocumentTypeRepo(), value.getType()); VarScope<FixedTypeRepo> repo_scope(_repo, repo); diff --git a/documentapi/CMakeLists.txt b/documentapi/CMakeLists.txt index 9261bcf9114..beeda4afeb4 100644 --- a/documentapi/CMakeLists.txt +++ b/documentapi/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog config_cloudconfig vespalib diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java index d22d7ecb550..f39b8db3689 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java @@ -313,6 +313,10 @@ public class VisitorParameters extends Parameters { sb.append(" Max total hits: ").append(maxTotalHits).append('\n'); sb.append(" Max buckets: ").append(maxBucketsPerVisitor).append('\n'); sb.append(" Priority: ").append(getPriority().toString()).append('\n'); + if (slices > 1) { + sb.append(" Slice ID: %d\n".formatted(sliceId)); + sb.append(" Slice count: %d\n".formatted(slices)); + } sb.append(')'); return sb.toString(); diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp index 70f6fc24d83..43082d9dbae 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp @@ -7,7 +7,6 @@ #include <vespa/slobrok/sbmirror.h> #include <vespa/fnet/transport.h> #include <vespa/fnet/frt/supervisor.h> -#include <vespa/fastos/thread.h> #include <vespa/log/log.h> LOG_SETUP(".externpolicy"); diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.cpp index 34d2b6d3369..b2b648545cc 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.cpp @@ -9,7 +9,6 @@ #include <vespa/slobrok/sbmirror.h> #include <vespa/fnet/frt/supervisor.h> #include <vespa/fnet/transport.h> -#include <vespa/fastos/thread.h> #include <thread> using slobrok::api::IMirrorAPI; diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/mirror_with_all.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/mirror_with_all.cpp index 0841c2ed32b..d425bfb6679 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/mirror_with_all.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/mirror_with_all.cpp @@ -5,17 +5,15 @@ #include <vespa/slobrok/sbmirror.h> #include <vespa/fnet/frt/supervisor.h> #include <vespa/fnet/transport.h> -#include <vespa/fastos/thread.h> namespace documentapi { MirrorAndStuff::MirrorAndStuff(const slobrok::ConfiguratorFactory & config) - : _threadPool(std::make_unique<FastOS_ThreadPool>()), - _transport(std::make_unique<FNET_Transport>()), - _orb(std::make_unique<FRT_Supervisor>(_transport.get())), - _mirror(std::make_unique<slobrok::api::MirrorAPI>(*_orb, config)) + : _transport(std::make_unique<FNET_Transport>()), + _orb(std::make_unique<FRT_Supervisor>(_transport.get())), + _mirror(std::make_unique<slobrok::api::MirrorAPI>(*_orb, config)) { - _transport->Start(_threadPool.get()); + _transport->Start(); } MirrorAndStuff::~MirrorAndStuff() { diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/mirror_with_all.h b/documentapi/src/vespa/documentapi/messagebus/policies/mirror_with_all.h index 05571c4420e..ed5dd459768 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/mirror_with_all.h +++ b/documentapi/src/vespa/documentapi/messagebus/policies/mirror_with_all.h @@ -4,7 +4,6 @@ #include <memory> -class FastOS_ThreadPool; class FNET_Transport; class FRT_Supervisor; namespace slobrok { class ConfiguratorFactory; } @@ -18,7 +17,6 @@ public: ~MirrorAndStuff(); slobrok::api::IMirrorAPI * mirror() { return _mirror.get(); } private: - std::unique_ptr<FastOS_ThreadPool> _threadPool; std::unique_ptr<FNET_Transport> _transport; std::unique_ptr<FRT_Supervisor> _orb; std::unique_ptr<slobrok::api::IMirrorAPI> _mirror; diff --git a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java index 680853ef687..bd2b057835c 100644 --- a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java +++ b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java @@ -47,8 +47,10 @@ import com.yahoo.vespa.documentgen.test.Book; import com.yahoo.vespa.documentgen.test.Book.Ss0; import com.yahoo.vespa.documentgen.test.Book.Ss1; import com.yahoo.vespa.documentgen.test.Common; +import com.yahoo.vespa.documentgen.test.Common2; import com.yahoo.vespa.documentgen.test.ConcreteDocumentFactory; import com.yahoo.vespa.documentgen.test.Music; +import com.yahoo.vespa.documentgen.test.Music2; import com.yahoo.vespa.documentgen.test.Music3; import com.yahoo.vespa.documentgen.test.Music4; import com.yahoo.vespa.documentgen.test.Parent; @@ -1024,5 +1026,36 @@ public class DocumentGenPluginTest { assertTrue(docType.hasImportedField("my_foo")); assertFalse(docType.hasImportedField("some_field_that_does_not_exist")); } + + @Test + public void subtypes_are_tagged_as_inheriting_supertypes() { + // music -> common + assertTrue(Music.type.isA("common")); + assertTrue(Music.type.inherits(Common.type)); + // ... but not common2 + assertFalse(Music.type.inherits(Common2.type)); + + // music3 -> (music2 -> common), common2 + assertTrue(Music3.type.isA("common")); + assertTrue(Music3.type.isA("common2")); + assertTrue(Music3.type.isA("music2")); + assertTrue(Music3.type.inherits(Common.type)); + assertTrue(Music3.type.inherits(Common2.type)); + assertTrue(Music3.type.inherits(Music2.type)); + // ... but not parent + assertFalse(Music3.type.isA("parent")); + assertFalse(Music3.type.inherits(Parent.type)); + + // music4 -> music3 -> (music2 -> common), common2 + assertTrue(Music4.type.inherits(Common.type)); + assertTrue(Music4.type.inherits(Common2.type)); + assertTrue(Music4.type.inherits(Music2.type)); + assertTrue(Music4.type.inherits(Music3.type)); + // ... but not music + assertFalse(Music4.type.inherits(Music.type)); + + // parent has no explicit inheritance + assertFalse(Parent.type.isA("common")); + } } diff --git a/eval/src/vespa/eval/eval/fast_value.hpp b/eval/src/vespa/eval/eval/fast_value.hpp index 2eaefa3670c..47f99d19055 100644 --- a/eval/src/vespa/eval/eval/fast_value.hpp +++ b/eval/src/vespa/eval/eval/fast_value.hpp @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "value_builder_factory.h" -#include "fast_addr_map.h" +#include "fast_value_index.h" #include "inline_operation.h" #include <vespa/eval/instruction/generic_join.h> #include <vespa/vespalib/stllike/hashtable.hpp> @@ -12,16 +12,6 @@ namespace vespalib::eval { //----------------------------------------------------------------------------- -// This is the class instructions will look for when optimizing sparse -// operations by calling inline functions directly. -struct FastValueIndex final : Value::Index { - FastAddrMap map; - FastValueIndex(size_t num_mapped_dims_in, const StringIdVector &labels, size_t expected_subspaces_in) - : map(num_mapped_dims_in, labels, expected_subspaces_in) {} - size_t size() const override { return map.size(); } - std::unique_ptr<View> create_view(ConstArrayRef<size_t> dims) const override; -}; - inline bool is_fast(const Value::Index &index) { return (std::type_index(typeid(index)) == std::type_index(typeid(FastValueIndex))); } diff --git a/eval/src/vespa/eval/eval/fast_value_index.h b/eval/src/vespa/eval/eval/fast_value_index.h new file mode 100644 index 00000000000..edf96490db6 --- /dev/null +++ b/eval/src/vespa/eval/eval/fast_value_index.h @@ -0,0 +1,24 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "value.h" +#include "fast_addr_map.h" + +namespace vespalib::eval { + +/* + * Tensor value index, used to map labels to dense subspace indexes. + * + * This is the class instructions will look for when optimizing sparse + * operations by calling inline functions directly. + */ +struct FastValueIndex final : Value::Index { + FastAddrMap map; + FastValueIndex(size_t num_mapped_dims_in, const StringIdVector &labels, size_t expected_subspaces_in) + : map(num_mapped_dims_in, labels, expected_subspaces_in) {} + size_t size() const override { return map.size(); } + std::unique_ptr<View> create_view(ConstArrayRef<size_t> dims) const override; +}; + +} diff --git a/fastos/.gitignore b/fastos/.gitignore deleted file mode 100644 index 54e2680a6d8..00000000000 --- a/fastos/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -*.ilk -*.pdb -.Build_completed -.Dist_completed -.Install_completed -.PreBuild_completed -bin -include -lib -update.log -Makefile diff --git a/fastos/CMakeLists.txt b/fastos/CMakeLists.txt deleted file mode 100644 index c17752e234c..00000000000 --- a/fastos/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_define_module( - LIBS - src/vespa/fastos - - TESTS - src/tests -) diff --git a/fastos/OWNERS b/fastos/OWNERS deleted file mode 100644 index 912f61c59b8..00000000000 --- a/fastos/OWNERS +++ /dev/null @@ -1,2 +0,0 @@ -baldersheim -arnej27959 diff --git a/fastos/README b/fastos/README deleted file mode 100644 index ed9afabffb8..00000000000 --- a/fastos/README +++ /dev/null @@ -1,4 +0,0 @@ -Old OS abstraction layer - -obsolete, to be replaced with implementations using -standard C++14 threads and newer unix networking APIs diff --git a/fastos/src/.gitignore b/fastos/src/.gitignore deleted file mode 100644 index 2e8e6fd906a..00000000000 --- a/fastos/src/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/Makefile.ini -/config_command.sh -/project.dsw diff --git a/fastos/src/tests/.gitignore b/fastos/src/tests/.gitignore deleted file mode 100644 index ccb5f0210ab..00000000000 --- a/fastos/src/tests/.gitignore +++ /dev/null @@ -1,17 +0,0 @@ -/Makefile -/backtracetest -/backtracetest.log -/filetest -/filetest.log -/processtest -/processtest.log -/sockettest -/sockettest.log -/threadtest -/threadtest.log -/timetest -/timetest.log -/typetest -/typetest.log -/usecputest -*test_app diff --git a/fastos/src/tests/CMakeLists.txt b/fastos/src/tests/CMakeLists.txt deleted file mode 100644 index 3bf68a88e79..00000000000 --- a/fastos/src/tests/CMakeLists.txt +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(fastos_filetest_app TEST - SOURCES - filetest.cpp - DEPENDS - fastos -) -vespa_add_test(NAME fastos_filetest_app NO_VALGRIND COMMAND fastos_filetest_app) -vespa_add_executable(fastos_thread_stats_test_app TEST - SOURCES - thread_stats_test.cpp - DEPENDS - fastos -) -vespa_add_test(NAME fastos_thread_stats_test_app NO_VALGRIND COMMAND fastos_thread_stats_test_app) -vespa_add_executable(fastos_thread_joinwait_test_app TEST - SOURCES - thread_joinwait_test.cpp - DEPENDS - fastos -) -vespa_add_test(NAME fastos_thread_joinwait_test_app NO_VALGRIND COMMAND fastos_thread_joinwait_test_app) -vespa_add_executable(fastos_threadtest_app TEST - SOURCES - threadtest.cpp - DEPENDS - fastos -) -vespa_add_test(NAME fastos_threadtest_app NO_VALGRIND COMMAND fastos_threadtest_app) -vespa_add_executable(fastos_typetest_app TEST - SOURCES - typetest.cpp - DEPENDS - fastos -) -vespa_add_test(NAME fastos_typetest_app NO_VALGRIND COMMAND fastos_typetest_app) diff --git a/fastos/src/tests/coretest2.cpp b/fastos/src/tests/coretest2.cpp deleted file mode 100644 index bd93623922b..00000000000 --- a/fastos/src/tests/coretest2.cpp +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - - - - -static void -bomb(void) -{ - char *p; - - p = nullptr; - *p = 4; -} - -void *BomberRun(void *arg) -{ - (void) arg; - bomb(); - return nullptr; -}; - -static int -bombMain(void) -{ - pthread_t thread; - void *ret; - - (void) pthread_create(&thread, nullptr, BomberRun, nullptr); - ret = nullptr; - (void) pthread_join(thread, &ret); - return (0); -} - - -int -main(int argc, char **argv) -{ - (void) argc; - (void) argv; - return bombMain(); -} diff --git a/fastos/src/tests/filetest.cpp b/fastos/src/tests/filetest.cpp deleted file mode 100644 index d0f8bbfd98b..00000000000 --- a/fastos/src/tests/filetest.cpp +++ /dev/null @@ -1,555 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "tests.h" -#include <vespa/fastos/file.h> -#include <memory> -#include <cassert> -#include <sys/mman.h> -#include <filesystem> - -class FileTest : public BaseTest -{ -public: - const std::string srcDir = getenv("SOURCE_DIRECTORY") ? getenv("SOURCE_DIRECTORY") : "."; - const std::string roFilename = srcDir + "/hello.txt"; - const std::string woFilename = "generated/writeonlytest.txt"; - const std::string rwFilename = "generated/readwritetest.txt"; - - void GetCurrentDirTest () - { - TestHeader ("Get Current Directory Test"); - - std::string currentDir = FastOS_File::getCurrentDirectory(); - - Progress(!currentDir.empty(), - "Current dir: %s", !currentDir.empty() ? - currentDir.c_str() : "<failed>"); - - bool dirrc = FastOS_File::SetCurrentDirectory(".."); - - std::string parentDir; - - if (dirrc) { - parentDir = FastOS_File::getCurrentDirectory(); - } - - Progress(dirrc && strcmp(currentDir.c_str(), parentDir.c_str()) != 0, - "Parent dir: %s", !parentDir.empty() ? - parentDir.c_str() : "<failed>"); - - dirrc = FastOS_File::SetCurrentDirectory(currentDir.c_str()); - - Progress(dirrc, "Changed back to working directory."); - - PrintSeparator(); - } - - void MemoryMapTest (int mmap_flags) - { - TestHeader ("Memory Map Test"); - - int i; - const int bufSize = 1000; - - std::filesystem::create_directory(std::filesystem::path("generated")); - FastOS_File file("generated/memorymaptest"); - - bool rc = file.OpenReadWrite(); - Progress(rc, "Opening file 'generated/memorymaptest'"); - - if (rc) { - char *buffer = new char [bufSize]; - for (i = 0; i < bufSize; i++) { - buffer[i] = i % 256; - } - ssize_t wroteB = file.Write2(buffer, bufSize); - Progress(wroteB == bufSize, "Writing %d bytes to file", bufSize); - - bool close_ok = file.Close(); - assert(close_ok); - file.enableMemoryMap(mmap_flags); - - rc = file.OpenReadOnly(); - - Progress(rc, "Opening file 'generated/memorymaptest' read-only"); - if (rc) { - bool mmapEnabled; - char *mmapBuffer = nullptr; - - mmapEnabled = file.IsMemoryMapped(); - mmapBuffer = static_cast<char *>(file.MemoryMapPtr(0)); - - Progress(rc, "Memory mapping %s", - mmapEnabled ? "enabled" : "disabled"); - Progress(rc, "Map address: 0x%p", mmapBuffer); - - if (mmapEnabled) { - rc = 0; - for (i = 0; i < bufSize; i++) { - rc |= (mmapBuffer[i] == i % 256); - } - Progress(rc, "Reading %d bytes from memory map", bufSize); - } - } - delete [] buffer; - } - std::filesystem::remove_all(std::filesystem::path("generated")); - PrintSeparator(); - } - - void DirectIOTest () - { - TestHeader ("Direct Disk IO Test"); - - int i; - const int bufSize = 40000; - - std::filesystem::create_directory(std::filesystem::path("generated")); - FastOS_File file("generated/diotest"); - - bool rc = file.OpenWriteOnly(); - Progress(rc, "Opening file 'generated/diotest' write-only"); - - if (rc) { - char *buffer = new char [bufSize]; - - for (i=0; i<bufSize; i++) { - buffer[i] = 'A' + (i % 17); - } - ssize_t wroteB = file.Write2(buffer, bufSize); - Progress(wroteB == bufSize, "Writing %d bytes to file", bufSize); - - bool close_ok = file.Close(); - assert(close_ok); - - if (rc) { - file.EnableDirectIO(); - - rc = file.OpenReadOnly(); - Progress(rc, "Opening file 'generated/diotest' read-only"); - if (rc) { - bool dioEnabled; - size_t memoryAlignment=0; - size_t transferGranularity=0; - size_t transferMaximum=0; - - dioEnabled = file.GetDirectIORestrictions(memoryAlignment, - transferGranularity, - transferMaximum); - - Progress(rc, "DirectIO %s", dioEnabled ? "enabled" : "disabled"); - Progress(rc, "Memory alignment: %u bytes", memoryAlignment); - Progress(rc, "Transfer granularity: %u bytes", transferGranularity); - Progress(rc, "Transfer maximum: %u bytes", transferMaximum); - - if (dioEnabled) { - int eachRead = (8192 + transferGranularity - 1) / transferGranularity; - - char *buffer2 = new char [(eachRead * transferGranularity + - memoryAlignment - 1)]; - char *alignPtr = buffer2; - unsigned int align = - static_cast<unsigned int> - (reinterpret_cast<unsigned long>(alignPtr) & - (memoryAlignment - 1)); - if (align != 0) { - alignPtr = &alignPtr[memoryAlignment - align]; - } - int residue = bufSize; - int pos=0; - while (residue > 0) { - int readThisTime = eachRead * transferGranularity; - if (readThisTime > residue) { - readThisTime = residue; - } - file.ReadBuf(alignPtr, readThisTime, pos); - - for (i=0; i<readThisTime; i++) { - rc = (alignPtr[i] == 'A' + ((i+pos) % 17)); - if (!rc) { - Progress(false, "Read error at offset %d", i); - break; - } - } - residue -= readThisTime; - pos += readThisTime; - - if (!rc) break; - } - if (rc) { - Progress(true, "Read success"); - - rc = file.SetPosition(1); - Progress(rc, "SetPosition(1)"); - if (rc) { - try { - const int attemptReadBytes = 173; - ssize_t readB = file.Read(buffer, attemptReadBytes); - Progress(false, "Expected to get an exception for unaligned read"); - ProgressI64(readB == attemptReadBytes, "Got %ld bytes from attempted 173", readB); - } catch(const DirectIOException &e) { - Progress(true, "Got exception as expected"); - } - } - if (rc) { - rc = file.SetPosition(1); - Progress(rc, "SetPosition(1)"); - if (rc) { - try { - const int attemptReadBytes = 4096; - ssize_t readB = file.Read(buffer, attemptReadBytes); - Progress(false, "Expected to get an exception for unaligned read"); - ProgressI64(readB == attemptReadBytes, "Got %ld bytes from attempted 4096", readB); - } catch(const DirectIOException &e) { - Progress(true, "Got exception as expected"); - } - } - } - } - delete [] buffer2; - } else { - memset(buffer, 0, bufSize); - - ssize_t readBytes = file.Read(buffer, bufSize); - Progress(readBytes == bufSize, - "Reading %d bytes from file", bufSize); - - for (i=0; i<bufSize; i++) { - rc = (buffer[i] == 'A' + (i % 17)); - if (!rc) { - Progress(false, "Read error at offset %d", i); - break; - } - } - if (rc) Progress(true, "Read success"); - } - } - } - delete [] buffer; - } - - std::filesystem::remove_all(std::filesystem::path("generated")); - PrintSeparator(); - } - - void ReadOnlyTest () - { - TestHeader("Read-Only Test"); - - FastOS_File *myFile = new FastOS_File(roFilename.c_str()); - - if (myFile->OpenReadOnly()) { - int64_t filesize; - filesize = myFile->GetSize(); - ProgressI64((filesize == 27), "File size: %ld", filesize); - - char dummyData[6] = "Dummy"; - bool writeResult = myFile->CheckedWrite(dummyData, 6); - - if (writeResult) { - Progress(false, "Should not be able to write a file opened for read-only access."); - } else { - char dummyData2[28]; - Progress(true, "Write failed with read-only access."); - - bool rc = myFile->SetPosition(1); - Progress(rc, "Setting position to 1"); - - if (rc) { - ssize_t readBytes; - int64_t filePosition; - readBytes = myFile->Read(dummyData2, 28); - - Progress(readBytes == 26, "Attempting to read 28 bytes, should get 26. Got: %d", readBytes); - - filePosition = myFile->GetPosition(); - Progress(filePosition == 27, "File position should now be 27. Was: %d", int(filePosition)); - - readBytes = myFile->Read(dummyData2, 6); - Progress(readBytes == 0, "We should now get 0 bytes. Read: %d bytes", readBytes); - - filePosition = myFile->GetPosition(); - Progress(filePosition == 27, "File position should now be 27. Was: %d", int(filePosition)); - } - } - } else { - Progress(false, "Unable to open file '%s'.", roFilename.c_str()); - } - delete(myFile); - PrintSeparator(); - } - - void WriteOnlyTest () - { - TestHeader("Write-Only Test"); - std::filesystem::create_directory(std::filesystem::path("generated")); - - FastOS_File *myFile = new FastOS_File(woFilename.c_str()); - - if (myFile->OpenWriteOnly()) { - int64_t filesize; - filesize = myFile->GetSize(); - - ProgressI64((filesize == 0), "File size: %ld", filesize); - - char dummyData[6] = "Dummy"; - bool writeResult = myFile->CheckedWrite(dummyData, 6); - - if (!writeResult) { - Progress(false, "Should be able to write to file opened for write-only access."); - } else { - Progress(true, "Write 6 bytes ok."); - - int64_t filePosition = myFile->GetPosition(); - if (filePosition == 6) { - Progress(true, "Fileposition is now 6."); - - if (myFile->SetPosition(0)) { - Progress(true, "SetPosition(0) success."); - filePosition = myFile->GetPosition(); - - if (filePosition == 0) { - Progress(true, "Fileposition is now 0."); - - int readBytes = myFile->Read(dummyData, 6); - - if (readBytes != 6) { - Progress(true, "Trying to read a write-only file should fail and it did."); - Progress(true, "Return code was: %d.", readBytes); - } else { - Progress(false, "Read on a file with write-only access should fail, but it didn't."); - } - } else { - ProgressI64(false, "Fileposition should be 6, but was %ld.", filePosition); - } - } else { - Progress(false, "SetPosition(0) failed"); - } - } else { - ProgressI64(false, "Fileposition should be 6, but was %ld.", filePosition); - } - } - bool closeResult = myFile->Close(); - Progress(closeResult, "Close file."); - } else { - Progress(false, "Unable to open file '%s'.", woFilename.c_str()); - } - - - bool deleteResult = myFile->Delete(); - Progress(deleteResult, "Delete file '%s'.", woFilename.c_str()); - - delete(myFile); - std::filesystem::remove_all(std::filesystem::path("generated")); - PrintSeparator(); - } - - void ReadWriteTest () - { - TestHeader("Read/Write Test"); - std::filesystem::create_directory(std::filesystem::path("generated")); - - FastOS_File *myFile = new FastOS_File(rwFilename.c_str()); - - if (myFile->OpenExisting()) { - Progress(false, "OpenExisting() should not work when '%s' does not exist.", rwFilename.c_str()); - bool close_ok = myFile->Close(); - assert(close_ok); - } else { - Progress(true, "OpenExisting() should fail when '%s' does not exist, and it did.", rwFilename.c_str()); - } - - if (myFile->OpenReadWrite()) { - int64_t filesize; - - filesize = myFile->GetSize(); - - ProgressI64((filesize == 0), "File size: %ld", filesize); - - char dummyData[6] = "Dummy"; - - bool writeResult = myFile->CheckedWrite(dummyData, 6); - - if (!writeResult) { - Progress(false, "Should be able to write to file opened for read/write access."); - } else { - Progress(true, "Write 6 bytes ok."); - - int64_t filePosition = myFile->GetPosition(); - - if (filePosition == 6) { - Progress(true, "Fileposition is now 6."); - - if (myFile->SetPosition(0)) { - Progress(true, "SetPosition(0) success."); - filePosition = myFile->GetPosition(); - - if (filePosition == 0) { - Progress(true, "Fileposition is now 0."); - - char dummyData2[7]; - int readBytes = myFile->Read(dummyData2, 6); - - if (readBytes == 6) { - Progress(true, "Reading 6 bytes worked."); - - int cmpResult = memcmp(dummyData, dummyData2, 6); - Progress((cmpResult == 0), "Comparing the written and read result.\n"); - - bool rc = myFile->SetPosition(1); - Progress(rc, "Setting position to 1"); - - if (rc) { - readBytes = myFile->Read(dummyData2, 7); - - Progress(readBytes == 5, "Attempting to read 7 bytes, should get 5. Got: %d", readBytes); - - filePosition = myFile->GetPosition(); - Progress(filePosition == 6, "File position should now be 6. Was: %d", int(filePosition)); - - readBytes = myFile->Read(dummyData2, 6); - Progress(readBytes == 0, "We should not be able to read any more. Read: %d bytes", readBytes); - - filePosition = myFile->GetPosition(); - Progress(filePosition == 6, "File position should now be 6. Was: %d", int(filePosition)); - } - } else { - Progress(false, "Reading 6 bytes failed."); - } - } else { - ProgressI64(false, "Fileposition should be 6, but was %ld.", filePosition); - } - } else { - Progress(false, "SetPosition(0) failed"); - } - } else { - ProgressI64(false, "Fileposition should be 6, but was %ld.", filePosition); - } - } - - bool closeResult = myFile->Close(); - Progress(closeResult, "Close file."); - } else { - Progress(false, "Unable to open file '%s'.", rwFilename.c_str()); - } - bool deleteResult = myFile->Delete(); - Progress(deleteResult, "Delete file '%s'.", rwFilename.c_str()); - - delete(myFile); - std::filesystem::remove_all(std::filesystem::path("generated")); - PrintSeparator(); - } - - void ScanDirectoryTest() - { - TestHeader("DirectoryScan Test"); - - FastOS_DirectoryScan *scanDir = new FastOS_DirectoryScan("."); - - while (scanDir->ReadNext()) { - const char *name = scanDir->GetName(); - bool isDirectory = scanDir->IsDirectory(); - bool isRegular = scanDir->IsRegular(); - - printf("%-30s %s\n", name, - isDirectory ? "DIR" : (isRegular ? "FILE" : "UNKN")); - } - - delete(scanDir); - PrintSeparator(); - } - - void ReadBufTest () - { - TestHeader("ReadBuf Test"); - - FastOS_File file(roFilename.c_str()); - - char buffer[20]; - - if (file.OpenReadOnly()) { - int64_t position = file.GetPosition(); - Progress(position == 0, "File pointer should be 0 after opening file"); - - ssize_t has_read = file.Read(buffer, 4); - Progress(has_read == 4, "Must read 4 bytes"); - buffer[4] = '\0'; - position = file.GetPosition(); - Progress(position == 4, "File pointer should be 4 after reading 4 bytes"); - Progress(strcmp(buffer, "This") == 0, "[This]=[%s]", buffer); - - file.ReadBuf(buffer, 6, 8); - buffer[6] = '\0'; - position = file.GetPosition(); - Progress(position == 4, "File pointer should still be 4 after ReadBuf"); - Progress(strcmp(buffer, "a test") == 0, "[a test]=[%s]", buffer); - } - - PrintSeparator(); - } - - void DiskFreeSpaceTest () - { - TestHeader("DiskFreeSpace Test"); - - int64_t freeSpace = FastOS_File::GetFreeDiskSpace(roFilename.c_str()); - ProgressI64(freeSpace != -1, "DiskFreeSpace using file ('hello.txt'): %ld MB.", freeSpace == -1 ? -1 : freeSpace/(1024*1024)); - freeSpace = FastOS_File::GetFreeDiskSpace("."); - ProgressI64(freeSpace != -1, "DiskFreeSpace using dir (.): %ld MB.", freeSpace == -1 ? -1 : freeSpace/(1024*1024)); - PrintSeparator(); - } - - void MaxLengthTest () - { - TestHeader ("Max Lengths Test"); - - int maxval = FastOS_File::GetMaximumFilenameLength("."); - Progress(maxval > 5 && maxval < (512*1024), - "Maximum filename length = %d", maxval); - - maxval = FastOS_File::GetMaximumPathLength("."); - Progress(maxval > 5 && maxval < (512*1024), - "Maximum path length = %d", maxval); - - PrintSeparator(); - } - - int Main () override - { - printf("This test should be run in the 'tests' directory.\n\n"); - printf("grep for the string '%s' to detect failures.\n\n", failString); - - GetCurrentDirTest(); - DirectIOTest(); - MaxLengthTest(); - DiskFreeSpaceTest(); - ReadOnlyTest(); - WriteOnlyTest(); - ReadWriteTest(); - ScanDirectoryTest(); - ReadBufTest(); - MemoryMapTest(0); -#ifdef __linux__ - MemoryMapTest(MAP_HUGETLB); -#endif - - PrintSeparator(); - printf("END OF TEST (%s)\n", _argv[0]); - - return allWasOk() ? 0 : 1; - } - FileTest(); - ~FileTest(); -}; - -FileTest::FileTest() { } -FileTest::~FileTest() { } - - -int main (int argc, char **argv) -{ - FileTest app; - - setvbuf(stdout, nullptr, _IOLBF, 8192); - return app.Entry(argc, argv); -} diff --git a/fastos/src/tests/job.h b/fastos/src/tests/job.h deleted file mode 100644 index 4546cfe1daa..00000000000 --- a/fastos/src/tests/job.h +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <mutex> -#include <condition_variable> - -enum JobCode -{ - PRINT_MESSAGE_AND_WAIT3MSEC, - INCREASE_NUMBER, - WAIT_FOR_BREAK_FLAG, - WAIT_FOR_THREAD_TO_FINISH, - TEST_ID, - SILENTNOP, - NOP -}; - -class Job -{ -private: - Job(const Job &); - Job &operator=(const Job&); - -public: - JobCode code; - char *message; - std::mutex *mutex; - std::condition_variable *condition; - FastOS_ThreadInterface *otherThread, *ownThread; - std::atomic<int> result; - FastOS_ThreadId _threadId; - - Job() - : code(NOP), - message(nullptr), - mutex(nullptr), - condition(nullptr), - otherThread(nullptr), - ownThread(nullptr), - result(-1), - _threadId() - { - } - - ~Job() - { - if(message != nullptr) - free(message); - } -}; diff --git a/fastos/src/tests/mazeserver.cpp b/fastos/src/tests/mazeserver.cpp deleted file mode 100644 index b62e98645f2..00000000000 --- a/fastos/src/tests/mazeserver.cpp +++ /dev/null @@ -1,4 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#define DO_MAZE_SERVER 1 - -#include "sockettest.cpp" diff --git a/fastos/src/tests/performancetest.cpp b/fastos/src/tests/performancetest.cpp deleted file mode 100644 index f566979957a..00000000000 --- a/fastos/src/tests/performancetest.cpp +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <stdlib.h> - -#include "tests.h" - -void PerformanceTest (char *buffer); - -int main (int argc, char **argv) -{ - (void)argc; - (void)argv; - - void (*test)(char *buffer) = PerformanceTest; - - test(nullptr); - return 0; -} - -void PerformanceTest (char *buffer) -{ - // Cause exception - *static_cast<char *>(nullptr) = 'e'; - -#if 1 - FastOS_File file("test.txt"); - - if(file.OpenReadOnly()) - { - file.Read(buffer, 20); - file.Write2(buffer, 20); - file.Read(buffer, 20); - file.Write2(buffer, 20); - file.Read(buffer, 20); - file.Write2(buffer, 20); - } -#else - - int filedes = open("test.txt", O_RDONLY, 0664); - - if(filedes != -1) - { - write(filedes, buffer, 20); - read(filedes, buffer, 20); - write(filedes, buffer, 20); - read(filedes, buffer, 20); - write(filedes, buffer, 20); - read(filedes, buffer, 20); - write(filedes, buffer, 20); - - close(filedes); - } -#endif -} - diff --git a/fastos/src/tests/tests.h b/fastos/src/tests/tests.h deleted file mode 100644 index 3a6f2ef9010..00000000000 --- a/fastos/src/tests/tests.h +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/fastos/thread.h> -#include <cstring> -#include <csignal> - -class BaseTest -{ -private: - BaseTest(const BaseTest&); - BaseTest &operator=(const BaseTest&); - - int totallen; - bool _allOkFlag; -public: - int _argc; - char **_argv; - - const char *okString; - const char *failString; - - BaseTest () - : totallen(70), - _allOkFlag(true), - _argc(0), - _argv(nullptr), - okString("SUCCESS"), - failString("FAILURE") - { - } - - virtual int Main() = 0; - - int Entry(int argc, char **argv) { - _argc = argc; - _argv = argv; - return Main(); - } - - virtual ~BaseTest() {}; - - bool allWasOk() const { return _allOkFlag; } - - void PrintSeparator () - { - for(int i=0; i<totallen; i++) printf("-"); - printf("\n"); - } - - virtual void PrintProgress (char *string) - { - printf("%s", string); - } -#define MAX_STR_LEN 3000 - bool Progress (bool result, const char *str) - { - char string[MAX_STR_LEN]; - snprintf(string, sizeof(string), "%s: %s\n", - result ? okString : failString, str); - PrintProgress(string); - if (! result) { _allOkFlag = false; } - return result; - } - - bool Progress (bool result, const char *str, int d1) - { - char string[MAX_STR_LEN-100]; - snprintf(string, sizeof(string), str, d1); - return Progress(result, string); - } - - bool Progress (bool result, const char *str, int d1, int d2) - { - char string[MAX_STR_LEN-100]; - snprintf(string, sizeof(string), str, d1, d2); - return Progress(result, string); - } - - bool Progress (bool result, const char *str, const char *s1) - { - char string[MAX_STR_LEN-100]; - snprintf(string, sizeof(string), str, s1); - return Progress(result, string); - } - - bool Progress (bool result, const char *str, const FastOS_ThreadInterface *s1) - { - char string[MAX_STR_LEN-100]; - snprintf(string, sizeof(string), str, s1); - return Progress(result, string); - } - - bool Progress (bool result, const char *str, const char *s1, const char *s2) - { - char string[MAX_STR_LEN-100]; - snprintf(string, sizeof(string), str, s1, s2); - return Progress(result, string); - } - - bool Progress (bool result, const char *str, const char *s1, int d1) - { - char string[MAX_STR_LEN-100]; - snprintf(string, sizeof(string), str, s1, d1); - return Progress(result, string); - } - - bool Progress (bool result, const char *str, int d1, const char *s1) - { - char string[MAX_STR_LEN-100]; - snprintf(string, sizeof(string), str, d1, s1); - return Progress(result, string); - } - - bool ProgressI64 (bool result, const char *str, int64_t val) - { - char string[MAX_STR_LEN-100]; - snprintf(string, sizeof(string), str, val); - return Progress(result, string); - } - - bool ProgressFloat (bool result, const char *str, float val) - { - char string[MAX_STR_LEN-100]; - snprintf(string, sizeof(string), str, val); - return Progress(result, string); - } - - void Ok (const char *string) - { - Progress(true, string); - } - - void Fail (const char *string) - { - Progress(false, string); - } - - void TestHeader (const char *string) - { - int len = strlen(string); - int leftspace = (totallen - len)/2 - 2; - int rightspace = totallen - 4 - len - leftspace; - int i; - - printf("\n\n"); - for(i=0; i<totallen; i++) printf("*"); - printf("\n**"); - for(i=0; i<leftspace; i++) printf(" "); //forgot printf-specifier.. - printf("%s", string); - for(i=0; i<rightspace; i++) printf(" "); - printf("**\n"); - for(i=0; i<totallen; i++) printf("*"); - printf("\n"); - } -}; diff --git a/fastos/src/tests/thread_joinwait_test.cpp b/fastos/src/tests/thread_joinwait_test.cpp deleted file mode 100644 index 6c7e8a7dc3c..00000000000 --- a/fastos/src/tests/thread_joinwait_test.cpp +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "tests.h" -#include "job.h" -#include "thread_test_base.hpp" - -class Thread_JoinWait_Test : public ThreadTestBase -{ - int Main () override; - - void SingleThreadJoinWaitMultipleTest(int variant) - { - bool rc=false; - - char testName[300]; - - snprintf(testName, sizeof(testName), "Single Thread Join Wait Multiple Test %d", variant); - TestHeader(testName); - - FastOS_ThreadPool pool; - - const int testThreads=5; - int lastThreadNum = testThreads-1; - int i; - - Job jobs[testThreads]; - - std::mutex jobMutex; - - // The mutex is used to pause the first threads until we have created - // the last one. - jobMutex.lock(); - - for(i=0; i<lastThreadNum; i++) - { - jobs[i].code = WAIT_FOR_THREAD_TO_FINISH; - jobs[i].mutex = &jobMutex; - jobs[i].ownThread = pool.NewThread(this, static_cast<void *>(&jobs[i])); - - rc = (jobs[i].ownThread != nullptr); - Progress(rc, "Creating Thread %d", i+1); - - if(!rc) - break; - } - - if (rc) - { - jobs[lastThreadNum].code = (((variant & 2) != 0) ? NOP : PRINT_MESSAGE_AND_WAIT3MSEC); - jobs[lastThreadNum].message = strdup("This is the thread that others wait for."); - - FastOS_ThreadInterface *lastThread; - - lastThread = pool.NewThread(this, - static_cast<void *> - (&jobs[lastThreadNum])); - - rc = (lastThread != nullptr); - Progress(rc, "Creating last thread"); - - if (rc) - { - for(i=0; i<lastThreadNum; i++) { - jobs[i].otherThread = lastThread; - } - } - } - - jobMutex.unlock(); - - if ((variant & 1) != 0) - { - for (i=0; i<lastThreadNum; i++) - { - Progress(true, "Waiting for thread %d to finish using Join()", i+1); - jobs[i].ownThread->Join(); - Progress(true, "Thread %d finished.", i+1); - } - } - - Progress(true, "Closing pool."); - pool.Close(); - Progress(true, "Pool closed."); - PrintSeparator(); - } - -}; - -int Thread_JoinWait_Test::Main () -{ - printf("grep for the string '%s' to detect failures.\n\n", failString); - time_t before = time(0); - - SingleThreadJoinWaitMultipleTest(0); - { time_t now = time(0); printf("[%ld seconds]\n", now-before); before = now; } - SingleThreadJoinWaitMultipleTest(1); - { time_t now = time(0); printf("[%ld seconds]\n", now-before); before = now; } - SingleThreadJoinWaitMultipleTest(2); - { time_t now = time(0); printf("[%ld seconds]\n", now-before); before = now; } - SingleThreadJoinWaitMultipleTest(3); - { time_t now = time(0); printf("[%ld seconds]\n", now-before); before = now; } - SingleThreadJoinWaitMultipleTest(2); - { time_t now = time(0); printf("[%ld seconds]\n", now-before); before = now; } - SingleThreadJoinWaitMultipleTest(1); - { time_t now = time(0); printf("[%ld seconds]\n", now-before); before = now; } - SingleThreadJoinWaitMultipleTest(0); - { time_t now = time(0); printf("[%ld seconds]\n", now-before); before = now; } - - printf("END OF TEST (%s)\n", _argv[0]); - return allWasOk() ? 0 : 1; -} - -int main (int argc, char **argv) -{ - Thread_JoinWait_Test app; - setvbuf(stdout, nullptr, _IOLBF, 8192); - return app.Entry(argc, argv); -} diff --git a/fastos/src/tests/thread_stats_test.cpp b/fastos/src/tests/thread_stats_test.cpp deleted file mode 100644 index 40c1199135c..00000000000 --- a/fastos/src/tests/thread_stats_test.cpp +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "tests.h" -#include "job.h" -#include "thread_test_base.hpp" - -class Thread_Stats_Test : public ThreadTestBase -{ - void ThreadStatsTest () - { - int inactiveThreads; - int activeThreads; - int startedThreads; - - TestHeader("Thread Statistics Test"); - - FastOS_ThreadPool pool; - Job job[2]; - - inactiveThreads = pool.GetNumInactiveThreads(); - Progress(inactiveThreads == 0, "Initial inactive threads = %d", inactiveThreads); - activeThreads = pool.GetNumActiveThreads(); - Progress(activeThreads == 0, "Initial active threads = %d", activeThreads); - startedThreads = pool.GetNumStartedThreads(); - Progress(startedThreads == 0, "Initial started threads = %d", startedThreads); - - job[0].code = WAIT_FOR_BREAK_FLAG; - job[0].ownThread = pool.NewThread(this, static_cast<void *>(&job[0])); - - inactiveThreads = pool.GetNumInactiveThreads(); - Progress(inactiveThreads == 0, "Inactive threads = %d", inactiveThreads); - activeThreads = pool.GetNumActiveThreads(); - Progress(activeThreads == 1, "Active threads = %d", activeThreads); - startedThreads = pool.GetNumStartedThreads(); - Progress(startedThreads == 1, "Started threads = %d", startedThreads); - - job[1].code = WAIT_FOR_BREAK_FLAG; - job[1].ownThread = pool.NewThread(this, static_cast<void *>(&job[1])); - - inactiveThreads = pool.GetNumInactiveThreads(); - Progress(inactiveThreads == 0, "Inactive threads = %d", inactiveThreads); - activeThreads = pool.GetNumActiveThreads(); - Progress(activeThreads == 2, "Active threads = %d", activeThreads); - startedThreads = pool.GetNumStartedThreads(); - Progress(startedThreads == 2, "Started threads = %d", startedThreads); - - Progress(true, "Setting breakflag on threads..."); - job[0].ownThread->SetBreakFlag(); - job[1].ownThread->SetBreakFlag(); - - job[0].ownThread->Join(); - job[1].ownThread->Join(); - while (pool.GetNumInactiveThreads() != 2) { - std::this_thread::sleep_for(1ms); - } - - inactiveThreads = pool.GetNumInactiveThreads(); - Progress(inactiveThreads == 2, "Inactive threads = %d", inactiveThreads); - activeThreads = pool.GetNumActiveThreads(); - Progress(activeThreads == 0, "Active threads = %d", activeThreads); - startedThreads = pool.GetNumStartedThreads(); - Progress(startedThreads == 2, "Started threads = %d", startedThreads); - - Progress(true, "Repeating process in the same pool..."); - - job[0].code = WAIT_FOR_BREAK_FLAG; - job[0].ownThread = pool.NewThread(this, static_cast<void *>(&job[0])); - - inactiveThreads = pool.GetNumInactiveThreads(); - Progress(inactiveThreads == 1, "Inactive threads = %d", inactiveThreads); - activeThreads = pool.GetNumActiveThreads(); - Progress(activeThreads == 1, "Active threads = %d", activeThreads); - startedThreads = pool.GetNumStartedThreads(); - Progress(startedThreads == 3, "Started threads = %d", startedThreads); - - job[1].code = WAIT_FOR_BREAK_FLAG; - job[1].ownThread = pool.NewThread(this, static_cast<void *>(&job[1])); - - inactiveThreads = pool.GetNumInactiveThreads(); - Progress(inactiveThreads == 0, "Inactive threads = %d", inactiveThreads); - activeThreads = pool.GetNumActiveThreads(); - Progress(activeThreads == 2, "Active threads = %d", activeThreads); - startedThreads = pool.GetNumStartedThreads(); - Progress(startedThreads == 4, "Started threads = %d", startedThreads); - - Progress(true, "Setting breakflag on threads..."); - job[0].ownThread->SetBreakFlag(); - job[1].ownThread->SetBreakFlag(); - - job[0].ownThread->Join(); - job[1].ownThread->Join(); - while (pool.GetNumInactiveThreads() != 2) { - std::this_thread::sleep_for(1ms); - } - - inactiveThreads = pool.GetNumInactiveThreads(); - Progress(inactiveThreads == 2, "Inactive threads = %d", inactiveThreads); - activeThreads = pool.GetNumActiveThreads(); - Progress(activeThreads == 0, "Active threads = %d", activeThreads); - startedThreads = pool.GetNumStartedThreads(); - Progress(startedThreads == 4, "Started threads = %d", startedThreads); - - pool.Close(); - Progress(true, "Pool closed."); - - PrintSeparator(); - } - - int Main () override; -}; - -int Thread_Stats_Test::Main () -{ - printf("grep for the string '%s' to detect failures.\n\n", failString); - time_t before = time(0); - - ThreadStatsTest(); - { time_t now = time(0); printf("[%ld seconds]\n", now-before); before = now; } - - printf("END OF TEST (%s)\n", _argv[0]); - return allWasOk() ? 0 : 1; -} - -int main (int argc, char **argv) -{ - Thread_Stats_Test app; - setvbuf(stdout, nullptr, _IOLBF, 8192); - return app.Entry(argc, argv); -} diff --git a/fastos/src/tests/thread_test_base.hpp b/fastos/src/tests/thread_test_base.hpp deleted file mode 100644 index eb994537f6e..00000000000 --- a/fastos/src/tests/thread_test_base.hpp +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <chrono> -#include <thread> - -static std::atomic<int64_t> number; -#define INCREASE_NUMBER_AMOUNT 10000 - -using namespace std::chrono_literals; - -class ThreadTestBase : public BaseTest, public FastOS_Runnable -{ -private: - std::mutex printMutex; - -public: - ThreadTestBase(void) - : printMutex() - { - } - virtual ~ThreadTestBase() {} - - void PrintProgress (char *string) override { - std::lock_guard<std::mutex> guard(printMutex); - BaseTest::PrintProgress(string); - } - - void Run (FastOS_ThreadInterface *thread, void *arg) override; - - void WaitForThreadsToFinish (Job *jobs, int count) { - int i; - - Progress(true, "Waiting for threads to finish..."); - for(;;) { - bool threadsFinished=true; - - for (i=0; i<count; i++) { - if (jobs[i].result == -1) { - threadsFinished = false; - break; - } - } - - std::this_thread::sleep_for(1us); - - if(threadsFinished) - break; - } - - Progress(true, "Threads finished"); - } -}; - - -void ThreadTestBase::Run (FastOS_ThreadInterface *thread, void *arg) -{ - if(arg == nullptr) - return; - - Job *job = static_cast<Job *>(arg); - char someStack[15*1024]; - - memset(someStack, 0, 15*1024); - - switch(job->code) - { - case SILENTNOP: - { - job->result = 1; - break; - } - - case NOP: - { - Progress(true, "Doing NOP"); - job->result = 1; - break; - } - - case PRINT_MESSAGE_AND_WAIT3MSEC: - { - Progress(true, "Thread printing message: [%s]", job->message); - job->result = strlen(job->message); - - std::this_thread::sleep_for(3ms); - break; - } - - case INCREASE_NUMBER: - { - int result; - - std::unique_lock<std::mutex> guard; - if(job->mutex != nullptr) { - guard = std::unique_lock<std::mutex>(*job->mutex); - } - - result = static_cast<int>(number.load(std::memory_order_relaxed)); - - int sleepOn = (INCREASE_NUMBER_AMOUNT/2) * 321/10000; - for (int i=0; i<(INCREASE_NUMBER_AMOUNT/2); i++) { - number.fetch_add(2, std::memory_order_relaxed); - - if (i == sleepOn) - std::this_thread::sleep_for(1ms); - } - - guard = std::unique_lock<std::mutex>(); - - job->result = result; // This marks the end of the thread - - break; - } - - case WAIT_FOR_BREAK_FLAG: - { - for(;;) { - std::this_thread::sleep_for(1us); - - if (thread->GetBreakFlag()) { - Progress(true, "Thread %p got breakflag", thread); - break; - } - } - break; - } - - case WAIT_FOR_THREAD_TO_FINISH: - { - std::unique_lock<std::mutex> guard; - if (job->mutex != nullptr) { - guard = std::unique_lock<std::mutex>(*job->mutex); - } - - if (job->otherThread != nullptr) - job->otherThread->Join(); - - break; - } - - case TEST_ID: - { - job->mutex->lock(); // Initially the parent threads owns the lock - job->mutex->unlock(); // It is unlocked when we should start - - FastOS_ThreadId currentId = FastOS_Thread::GetCurrentThreadId(); - - if(currentId == job->_threadId) - job->result = 1; - else - job->result = -1; - break; - } - - default: - Progress(false, "Unknown jobcode"); - break; - } -} diff --git a/fastos/src/tests/threadtest.cpp b/fastos/src/tests/threadtest.cpp deleted file mode 100644 index 563b41ac229..00000000000 --- a/fastos/src/tests/threadtest.cpp +++ /dev/null @@ -1,283 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "tests.h" -#include "job.h" -#include "thread_test_base.hpp" -#include <cstdlib> -#include <chrono> - -#define MAX_THREADS 7 - -using namespace std::chrono; -using namespace std::chrono_literals; - -class ThreadTest : public ThreadTestBase -{ - int Main () override; - - void TooManyThreadsTest () - { - TestHeader("Too Many Threads Test"); - static constexpr size_t message_size = 100; - - FastOS_ThreadPool *pool = new FastOS_ThreadPool(MAX_THREADS); - - if (Progress(pool != nullptr, "Allocating ThreadPool")) { - int i; - Job jobs[MAX_THREADS+1]; - - for (i=0; i<MAX_THREADS+1; i++) { - jobs[i].code = WAIT_FOR_BREAK_FLAG; - jobs[i].message = static_cast<char *>(malloc(message_size)); - if (jobs[i].message == nullptr) { - abort(); // GCC may infer that a potentially null ptr is passed to sprintf - } - snprintf(jobs[i].message, message_size, "Thread %d invocation", i+1); - } - - for (i=0; i<MAX_THREADS+1; i++) { - if (i==MAX_THREADS) { - while (pool->GetNumInactiveThreads() > 0); - jobs[MAX_THREADS].code = PRINT_MESSAGE_AND_WAIT3MSEC; - bool rc = (nullptr == pool->NewThread(this, static_cast<void *>(&jobs[MAX_THREADS]))); - Progress(rc, "Creating too many threads should fail."); - } else { - jobs[i].ownThread = pool->NewThread(this, static_cast<void *>(&jobs[i])); - Progress(jobs[i].ownThread != nullptr, "Creating Thread"); - } - } - for (i=0; i<MAX_THREADS; i++) { - jobs[i].ownThread->SetBreakFlag(); - } - - Progress(true, "Closing threadpool..."); - pool->Close(); - - Progress(true, "Deleting threadpool..."); - delete(pool); - } - PrintSeparator(); - } - - void CreateSingleThreadAndJoin () - { - TestHeader("Create Single Thread And Join Test"); - - FastOS_ThreadPool *pool = new FastOS_ThreadPool; - - if (Progress(pool != nullptr, "Allocating ThreadPool")) { - Job job; - - job.code = NOP; - job.result = -1; - - bool rc = (nullptr != pool->NewThread(this, &job)); - Progress(rc, "Creating Thread"); - - WaitForThreadsToFinish(&job, 1); - } - - Progress(true, "Closing threadpool..."); - pool->Close(); - - Progress(true, "Deleting threadpool..."); - delete(pool); - PrintSeparator(); - } - - void ThreadCreatePerformance (bool silent, int count, int outercount) { - int i; - int j; - bool rc; - int threadsok; - - if (!silent) - TestHeader("Thread Create Performance"); - - FastOS_ThreadPool *pool = new FastOS_ThreadPool; - - if (!silent) - Progress(pool != nullptr, "Allocating ThreadPool"); - if (pool != nullptr) { - Job *jobs = new Job[count]; - - threadsok = 0; - steady_clock::time_point start = steady_clock::now(); - for (i = 0; i < count; i++) { - jobs[i].code = SILENTNOP; - jobs[i].ownThread = pool->NewThread(this, &jobs[i]); - rc = (jobs[i].ownThread != nullptr); - if (rc) - threadsok++; - } - - for (j = 0; j < outercount; j++) { - for (i = 0; i < count; i++) { - if (jobs[i].ownThread != nullptr) - jobs[i].ownThread->Join(); - jobs[i].ownThread = pool->NewThread(this, &jobs[i]); - rc = (jobs[i].ownThread != nullptr); - if (rc) - threadsok++; - } - } - for (i = 0; i < count; i++) { - if (jobs[i].ownThread != nullptr) - jobs[i].ownThread->Join(); - } - nanoseconds used = steady_clock::now() - start; - - if (!silent) { - double timeused = used.count() / 1000000000.0; - - Progress(true, "Used time: %2.3f", timeused); - ProgressFloat(true, "Threads/s: %6.1f", - static_cast<float>(static_cast<double>(threadsok) / timeused)); - } - if (threadsok != ((outercount + 1) * count)) - Progress(false, "Only started %d of %d threads", threadsok, - (outercount + 1) * count); - - if (!silent) - Progress(true, "Closing threadpool..."); - pool->Close(); - delete [] jobs; - - if (!silent) - Progress(true, "Deleting threadpool..."); - delete(pool); - if (!silent) - PrintSeparator(); - } - } - - void ClosePoolStability(void) { - int i; - TestHeader("ThreadPool close stability test"); - for (i = 0; i < 1000; i++) { - // Progress(true, "Creating pool iteration %d", i + 1); - ThreadCreatePerformance(true, 2, 1); - } - PrintSeparator(); - } - - - - void ClosePoolTest () - { - TestHeader("Close Pool Test"); - - FastOS_ThreadPool pool; - const int closePoolThreads=9; - Job jobs[closePoolThreads]; - - number = 0; - - for(int i=0; i<closePoolThreads; i++) { - jobs[i].code = INCREASE_NUMBER; - - bool rc = (nullptr != pool.NewThread(this, static_cast<void *>(&jobs[i]))); - Progress(rc, "Creating Thread %d", i+1); - } - - Progress(true, "Waiting for threads to finish using pool.Close()..."); - pool.Close(); - Progress(true, "Pool closed."); - PrintSeparator(); - } - - void BreakFlagTest () { - TestHeader("BreakFlag Test"); - - FastOS_ThreadPool pool; - - const int breakFlagThreads=4; - - Job jobs[breakFlagThreads]; - - for (int i=0; i<breakFlagThreads; i++) { - jobs[i].code = WAIT_FOR_BREAK_FLAG; - - bool rc = (nullptr != pool.NewThread(this, static_cast<void *>(&jobs[i]))); - Progress(rc, "Creating Thread %d", i+1); - } - - Progress(true, "Waiting for threads to finish using pool.Close()..."); - pool.Close(); - Progress(true, "Pool closed."); - PrintSeparator(); - } - - void ThreadIdTest () { - constexpr int numThreads = 5; - - TestHeader ("Thread Id Test"); - - FastOS_ThreadPool pool; - Job jobs[numThreads]; - std::mutex slowStartMutex; - - slowStartMutex.lock(); // Halt all threads until we want them to run - - for (int i=0; i<numThreads; i++) { - jobs[i].code = TEST_ID; - jobs[i].result = -1; - jobs[i]._threadId = 0; - jobs[i].mutex = &slowStartMutex; - jobs[i].ownThread = pool.NewThread(this, static_cast<void *>(&jobs[i])); - bool rc=(jobs[i].ownThread != nullptr); - if (rc) { - jobs[i]._threadId = jobs[i].ownThread->GetThreadId(); - } - Progress(rc, "CreatingThread %d id:%lu", i+1, (unsigned long)(jobs[i]._threadId)); - - for (int j=0; j<i; j++) { - if (jobs[j]._threadId == jobs[i]._threadId) { - Progress(false, "Two different threads received the same thread id (%lu)", - (unsigned long)(jobs[i]._threadId)); - } - } - } - - slowStartMutex.unlock(); // Allow threads to run - - Progress(true, "Waiting for threads to finish using pool.Close()..."); - pool.Close(); - Progress(true, "Pool closed."); - - for (int i=0; i<numThreads; i++) { - Progress(jobs[i].result == 1, - "Thread %lu: ID comparison (current vs stored)", - (unsigned long)(jobs[i]._threadId)); - } - - PrintSeparator(); - } - -}; - -int ThreadTest::Main () -{ - printf("grep for the string '%s' to detect failures.\n\n", failString); - time_t before = time(0); - - ThreadIdTest(); - CreateSingleThreadAndJoin(); - TooManyThreadsTest(); - ClosePoolTest(); - BreakFlagTest(); - CreateSingleThreadAndJoin(); - ThreadCreatePerformance(false, 50, 10); - ClosePoolStability(); - { time_t now = time(0); printf("[%ld seconds]\n", now-before); before = now; } - - printf("END OF TEST (%s)\n", _argv[0]); - return allWasOk() ? 0 : 1; -} - -int main (int argc, char **argv) -{ - ThreadTest app; - setvbuf(stdout, nullptr, _IOLBF, 8192); - return app.Entry(argc, argv); -} diff --git a/fastos/src/tests/typetest.cpp b/fastos/src/tests/typetest.cpp deleted file mode 100644 index e5d7e9ceb74..00000000000 --- a/fastos/src/tests/typetest.cpp +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "tests.h" -#include <vespa/fastos/file.h> - -class TypeTest : public BaseTest -{ -private: - - void ObjectSizeTest () - { - TestHeader("Object Sizes (bytes)"); - - Progress(true, "FastOS_DirectoryScan %d", sizeof(FastOS_DirectoryScan)); - Progress(true, "FastOS_File: %d", sizeof(FastOS_File)); - Progress(true, "FastOS_Runnable %d", sizeof(FastOS_Runnable)); - Progress(true, "FastOS_StatInfo %d", sizeof(FastOS_StatInfo)); - Progress(true, "FastOS_Thread: %d", sizeof(FastOS_Thread)); - Progress(true, "FastOS_ThreadPool: %d", sizeof(FastOS_ThreadPool)); - - PrintSeparator(); - } - -public: - virtual ~TypeTest() {}; - - int Main () override - { - printf("grep for the string '%s' to detect failures.\n\n", failString); - - ObjectSizeTest(); - - PrintSeparator(); - printf("END OF TEST (%s)\n", _argv[0]); - - return allWasOk() ? 0 : 1; - } -}; - - -int main (int argc, char **argv) -{ - setvbuf(stdout, nullptr, _IOLBF, 8192); - TypeTest app; - return app.Entry(argc, argv); -} - diff --git a/fastos/src/vespa/fastos/.gitignore b/fastos/src/vespa/fastos/.gitignore deleted file mode 100644 index 004799df5b4..00000000000 --- a/fastos/src/vespa/fastos/.gitignore +++ /dev/null @@ -1,28 +0,0 @@ -*.So -*.core -*.exe -*.ilk -*.pdb -.depend -.depend.NEW -Debug -Makefile -Makefile.factory -Makefile.overrides -Makefile.pre -Release -autoconf.h -config_command.bat -config_command.sh -fastconfig -fastconfig.exe -fastos.lib -fastosconfig.h -libfastos.a -makefeatures -makemake -processtest.log -test.txt -vc60.idb -vc60.pdb -/libfastos.so.5.1 diff --git a/fastos/src/vespa/fastos/CMakeLists.txt b/fastos/src/vespa/fastos/CMakeLists.txt deleted file mode 100644 index 0e2bb51f79e..00000000000 --- a/fastos/src/vespa/fastos/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(fastos_objects OBJECT - SOURCES - file.cpp - file_rw_ops.cpp - linux_file.cpp - thread.cpp - unix_file.cpp - unix_thread.cpp -) - -vespa_add_library(fastos - SOURCES - $<TARGET_OBJECTS:fastos_objects> - INSTALL lib64 - DEPENDS - ${CMAKE_DL_LIBS} -) - -find_package(Threads REQUIRED) -target_link_libraries(fastos PUBLIC ${CMAKE_THREAD_LIBS_INIT}) diff --git a/fastos/src/vespa/fastos/thread.cpp b/fastos/src/vespa/fastos/thread.cpp deleted file mode 100644 index 9a9c3321cac..00000000000 --- a/fastos/src/vespa/fastos/thread.cpp +++ /dev/null @@ -1,363 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -//************************************************************************ -/** - * Implementation of FastOS_ThreadPool and FastOS_Thread methods. - * - * @author Oivind H. Danielsen - */ - -#include "thread.h" -#include <cstdio> -#include <cassert> - -// ---------------------------------------------------------------------- -// FastOS_ThreadPool -// ---------------------------------------------------------------------- - -FastOS_ThreadPool::FastOS_ThreadPool() : FastOS_ThreadPool(0) {} - -FastOS_ThreadPool::FastOS_ThreadPool(int maxThreads) - : _startedThreadsCount(0), - _closeFlagMutex(), - _closeCalledFlag(false), - _freeMutex(), - _liveMutex(), - _liveCond(), - _freeThreads(nullptr), - _activeThreads(nullptr), - _numFree(0), - _numActive(0), - _numTerminated(0), - _numLive(0), - _maxThreads(maxThreads) // 0 means unbounded -{ -} - -FastOS_ThreadPool::~FastOS_ThreadPool(void) -{ - Close(); -} - -void FastOS_ThreadPool::ThreadIsAboutToTerminate(FastOS_ThreadInterface *) -{ - assert(isClosed()); - - std::lock_guard<std::mutex> guard(_liveMutex); - - _numTerminated++; - _numLive--; - if (_numLive == 0) { - _liveCond.notify_all(); - } -} - - -// This is a NOP if the thread isn't active. -void FastOS_ThreadPool::FreeThread (FastOS_ThreadInterface *thread) -{ - std::lock_guard<std::mutex> guard(_freeMutex); - - if(thread->_active) { - LinkOutThread(thread, &_activeThreads); - - thread->_active = false; - _numActive--; - - LinkInThread(thread, &_freeThreads); - _numFree++; - } -} - -void FastOS_ThreadPool::LinkOutThread (FastOS_ThreadInterface *thread, FastOS_ThreadInterface **listHead) -{ - if (thread->_prev != nullptr) - thread->_prev->_next = thread->_next; - if (thread->_next != nullptr) - thread->_next->_prev = thread->_prev; - - if (thread == *listHead) - *listHead = thread->_next; -} - -void FastOS_ThreadPool::LinkInThread (FastOS_ThreadInterface *thread, FastOS_ThreadInterface **listHead) -{ - thread->_prev = nullptr; - thread->_next = *listHead; - - if (*listHead != nullptr) - (*listHead)->_prev = thread; - - *listHead = thread; -} - - -// _freeMutex is held by caller. -void FastOS_ThreadPool::ActivateThread (FastOS_ThreadInterface *thread) -{ - LinkOutThread(thread, &_freeThreads); - LinkInThread(thread, &_activeThreads); - - thread->_active = true; - _numActive++; - _startedThreadsCount++; -} - - -// Allocate a thread, either from pool of free or by 'new'. Finally, -// make this thread call parameter fcn when it becomes active. -FastOS_ThreadInterface *FastOS_ThreadPool::NewThread (FastOS_Runnable *owner, void *arg) -{ - FastOS_ThreadInterface *thread=nullptr; - - std::unique_lock<std::mutex> freeGuard(_freeMutex); - - if (!isClosed()) { - if ((thread = _freeThreads) != nullptr) { - // Reusing thread entry - _freeThreads = thread->_next; - _numFree--; - - ActivateThread(thread); - } else { - // Creating new thread entry - - if (_maxThreads != 0 && ((_numActive + _numFree) >= _maxThreads)) { - fprintf(stderr, "Error: Maximum number of threads (%d)" - " already allocated.\n", _maxThreads); - } else { - freeGuard.unlock(); - { - std::lock_guard<std::mutex> liveGuard(_liveMutex); - _numLive++; - } - thread = FastOS_Thread::CreateThread(this); - - if (thread == nullptr) { - std::lock_guard<std::mutex> liveGuard(_liveMutex); - _numLive--; - if (_numLive == 0) { - _liveCond.notify_all(); - } - } - freeGuard.lock(); - - if(thread != nullptr) - ActivateThread(thread); - } - } - } - - freeGuard.unlock(); - if(thread != nullptr) { - std::lock_guard<std::mutex> liveGuard(_liveMutex); - thread->Dispatch(owner, arg); - } - - return thread; -} - - -void FastOS_ThreadPool::BreakThreads () -{ - FastOS_ThreadInterface *thread; - - std::lock_guard<std::mutex> freeGuard(_freeMutex); - - // Notice all active threads that they should quit - for(thread=_activeThreads; thread != nullptr; thread=thread->_next) { - thread->SetBreakFlag(); - } - - // Notice all free threads that they should quit - for(thread=_freeThreads; thread != nullptr; thread=thread->_next) { - thread->SetBreakFlag(); - } -} - - -void FastOS_ThreadPool::JoinThreads () -{ - std::unique_lock<std::mutex> liveGuard(_liveMutex); - while (_numLive > 0) { - _liveCond.wait(liveGuard); - } -} - -void FastOS_ThreadPool::DeleteThreads () -{ - FastOS_ThreadInterface *thread; - - std::lock_guard<std::mutex> freeGuard(_freeMutex); - - assert(_numActive == 0); - assert(_numLive == 0); - - while((thread = _freeThreads) != nullptr) { - LinkOutThread(thread, &_freeThreads); - _numFree--; - // printf("deleting thread %p\n", thread); - delete(thread); - } - - assert(_numFree == 0); -} - -void FastOS_ThreadPool::Close () -{ - std::unique_lock<std::mutex> closeFlagGuard(_closeFlagMutex); - if (!_closeCalledFlag) { - _closeCalledFlag = true; - closeFlagGuard.unlock(); - - BreakThreads(); - JoinThreads(); - DeleteThreads(); - } -} - -bool FastOS_ThreadPool::isClosed() -{ - std::lock_guard<std::mutex> closeFlagGuard(_closeFlagMutex); - bool closed(_closeCalledFlag); - return closed; -} - -extern "C" -{ -void *FastOS_ThreadHook (void *arg) -{ - FastOS_ThreadInterface *thread = static_cast<FastOS_ThreadInterface *>(arg); - thread->Hook(); - - return nullptr; -} -}; - - -// ---------------------------------------------------------------------- -// FastOS_ThreadInterface -// ---------------------------------------------------------------------- - -void FastOS_ThreadInterface::Hook () -{ - // Loop forever doing the following: Wait on the signal _dispatched. - // When awoken, call _start_fcn with the parameters. Then zero set - // things and return this to the owner, i.e. pool of free threads - bool finished=false; - bool deleteOnCompletion = false; - - while(!finished) { - - std::unique_lock<std::mutex> dispatchedGuard(_dispatchedMutex); // BEGIN lock - while (_owner == nullptr && !(finished = _pool->isClosed())) { - _dispatchedCond.wait(dispatchedGuard); - } - - dispatchedGuard.unlock(); // END lock - - if(!finished) { - deleteOnCompletion = _owner->DeleteOnCompletion(); - _owner->Run(this, _startArg); - - dispatchedGuard.lock(); // BEGIN lock - - if (deleteOnCompletion) { - delete _owner; - } - _owner = nullptr; - _startArg = nullptr; - _breakFlag.store(false, std::memory_order_relaxed); - finished = _pool->isClosed(); - - dispatchedGuard.unlock(); // END lock - - { - std::lock_guard<std::mutex> runningGuard(_runningMutex); - _runningFlag = false; - _runningCond.notify_all(); - } - - _pool->FreeThread(this); - // printf("Thread given back to FastOS_ThreadPool: %p\n", this); - } - } - - _pool->ThreadIsAboutToTerminate(this); - - // Be sure not to touch any members from here on, as we are about - // to be deleted. -} - - -// Make this thread call parameter fcn with parameters argh -// when this becomes active. -// Restriction: _liveCond must be held by the caller. - -void FastOS_ThreadInterface::Dispatch(FastOS_Runnable *newOwner, void *arg) -{ - std::lock_guard<std::mutex> dispatchedGuard(_dispatchedMutex); - - { - std::unique_lock<std::mutex> runningGuard(_runningMutex); - while (_runningFlag) { - _runningCond.wait(runningGuard); - } - _runningFlag = true; - } - - _owner = newOwner; - _startArg = arg; - // Set _thread variable before NewThread returns - _owner->_thread.store(this, std::memory_order_release); - - // It is safe to signal after the unlock since _liveCond is still held - // so the signalled thread still exists. - // However as thread creation is infrequent and as helgrind suggest doing - // it the safe way we just do that, instead of keeping a unneccessary long - // suppressionslist. It will be long enough anyway. - - _dispatchedCond.notify_one(); -} - -void FastOS_ThreadInterface::SetBreakFlag() -{ - std::lock_guard<std::mutex> dispatchedGuard(_dispatchedMutex); - _breakFlag.store(true, std::memory_order_relaxed); - _dispatchedCond.notify_one(); -} - - -FastOS_ThreadInterface *FastOS_ThreadInterface::CreateThread(FastOS_ThreadPool *pool) -{ - FastOS_ThreadInterface *thread = new FastOS_Thread(pool); - - if(!thread->Initialize()) { - delete(thread); - thread = nullptr; - } - - return thread; -} - -void FastOS_ThreadInterface::Join () -{ - std::unique_lock<std::mutex> runningGuard(_runningMutex); - while (_runningFlag) { - _runningCond.wait(runningGuard); - } -} - - -// ---------------------------------------------------------------------- -// FastOS_Runnable -// ---------------------------------------------------------------------- - -FastOS_Runnable::FastOS_Runnable() - : _thread(nullptr) -{ -} - -FastOS_Runnable::~FastOS_Runnable() -{ - // assert(_thread == nullptr); -} diff --git a/fastos/src/vespa/fastos/thread.h b/fastos/src/vespa/fastos/thread.h deleted file mode 100644 index 2fb717403f2..00000000000 --- a/fastos/src/vespa/fastos/thread.h +++ /dev/null @@ -1,467 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -//************************************************************************ -/** - * @file - * Class definitions for FastOS_ThreadPool, FastOS_ThreadInterface and - * FastOS_Runnable. - * - * @author Oivind H. Danielsen - */ - -#pragma once - - -#include "types.h" -#include <atomic> -#include <mutex> -#include <condition_variable> - -typedef pthread_t FastOS_ThreadId; - -class FastOS_Runnable; -class FastOS_ThreadInterface; - - -/** - * This class implements an initially empty pool of threads. - * - * As threads are allocated with @ref NewThread() the number of - * threads in the pool increases. A maximum number of threads - * contained in the pool can be set using the constructor - * FastOS_ThreadPool(int maxThreads). - * - * Threads are automatically returned to the pool when they - * terminate. - */ -class FastOS_ThreadPool -{ - friend class FastOS_ThreadInterface; - -private: - int _startedThreadsCount; - std::mutex _closeFlagMutex; - bool _closeCalledFlag; - - // Always lock in this order - mutable std::mutex _freeMutex; - std::mutex _liveMutex; - std::condition_variable _liveCond; - /** - * List of free (available) threads. - */ - FastOS_ThreadInterface *_freeThreads; - - /** - * List of active (allocated) threads. - */ - FastOS_ThreadInterface *_activeThreads; - - /** - * Number of available threads in the threadpool. - * Total number of threads = free + active - */ - int _numFree; - - /** - * Number of active threads in the threadpool. - * Total number of threads = free + active - */ - int _numActive; - - /** - * Number of threads that have terminated - */ - int _numTerminated; - - /** - * Number of threads that have not been terminated - */ - int _numLive; - - /** - * Maximum number of threads in the threadpool. A value of - * zero means that there is no limit. - */ - int _maxThreads; - - /** - * Put this thread on the @ref _activeThreads list. - */ - void ActivateThread (FastOS_ThreadInterface *thread); - - /** - * Return previously active thread to the list of free thread. - */ - void FreeThread (FastOS_ThreadInterface *thread); - - /** - * A thread is informing the thread pool that it is about to - * terminate. - */ - void ThreadIsAboutToTerminate(FastOS_ThreadInterface *thread); - - /** - * Set the break flag on all threads. - */ - void BreakThreads (); - - /** - * Wait for all threads to finish. - */ - void JoinThreads (); - - /** - * Delete all threads in threadpool. - */ - void DeleteThreads (); - - /** - * Remove a thread from a list. - */ - void LinkOutThread (FastOS_ThreadInterface *thread, - FastOS_ThreadInterface **listHead); - - /** - * Add a thread to a list. Notice that a thread can be on only one - * list at a time. - */ - void LinkInThread (FastOS_ThreadInterface *thread, - FastOS_ThreadInterface **listHead); - -public: - FastOS_ThreadPool(const FastOS_ThreadPool&) = delete; - FastOS_ThreadPool& operator=(const FastOS_ThreadPool&) = delete; - FastOS_ThreadPool(int maxThreads); - /// Unlimited threads - FastOS_ThreadPool(); - - /** - * Destructor. Closes pool if necessary. - */ - virtual ~FastOS_ThreadPool(); - - - /** - * Allocate a new thread, and make this thread invoke the Run() method - * of the @ref FastOS_Runnable object [owner] with parameters [arg]. - * The thread is automatically freed (returned to the treadpool) - * when Run() returns. - * - * @param owner Instance to be invoked by new thread. - * @param arg Arguments to be passed to new thread. - * - * @return Pointer to newly created thread or nullptr on failure. - */ - FastOS_ThreadInterface *NewThread (FastOS_Runnable *owner, void *arg=nullptr); - - /** - * Close the threadpool. This involves setting the break flag on - * all active threads, and waiting for them to finish. Once Close - * is called, no more threads can be allocated from the thread - * pool. There exists no way to reopen a closed threadpool. - */ - void Close (); - - /** - * This will tell if the pool has been closed. - */ - bool isClosed(); - - /** - * Get the number of currently active threads. - * The total number of actual allocated threads is the sum of - * @ref GetNumActiveThreads() and @ref GetNumInactiveThreads(). - * @return Number of currently active threads - */ - int GetNumActiveThreads () const { - std::lock_guard<std::mutex> guard(_freeMutex); - return _numActive; - } - - /** - * Get the number of currently inactive threads. - * The total number of actual allocated threads is the sum of - * @ref GetNumActiveThreads() and @ref GetNumInactiveThreads(). - * @return Number of currently inactive threads - */ - int GetNumInactiveThreads () const { - std::lock_guard<std::mutex> guard(_freeMutex); - return _numFree; - } - - /** - * Get the number of started threads since instantiation of the thread pool. - * @return Number of threads started - */ - int GetNumStartedThreads () const { return _startedThreadsCount; } -}; - - -// Operating system thread entry point -extern "C" { - void *FastOS_ThreadHook (void *arg); -} - -/** - * This class controls each operating system thread. - * - * In most cases you would not want to create objects of this class - * directly. Use @ref FastOS_ThreadPool::NewThread() instead. - */ -class FastOS_ThreadInterface -{ - friend class FastOS_ThreadPool; - friend void *FastOS_ThreadHook (void *arg); - -private: - FastOS_ThreadInterface(const FastOS_ThreadInterface&); - FastOS_ThreadInterface& operator=(const FastOS_ThreadInterface&); - -protected: - /** - * The thread does not start (call @ref FastOS_Runnable::Run()) - * until this event has been triggered. - */ - std::mutex _dispatchedMutex; - std::condition_variable _dispatchedCond; - - FastOS_ThreadInterface *_next; - FastOS_ThreadInterface *_prev; - - /** - * A pointer to the instance which implements the interface - * @ref FastOS_Runnable. - */ - FastOS_Runnable *_owner; - - /** - * A pointer to the originating @ref FastOS_ThreadPool - */ - FastOS_ThreadPool *_pool; - - /** - * Entry point for the OS thread. The thread will sleep here - * until dispatched. - */ - void Hook (); - - /** - * Signals that thread should be dispatched. - * @param owner Instance of @ref FastOS_Runnable. - * @param arg Thread invocation arguments. - */ - void Dispatch (FastOS_Runnable *owner, void *arg); - - /** - * Initializes a thread. This includes creating the operating system - * thread handle and setting it up and making it ready to be dispatched. - * @return Boolean success/failure - */ - virtual bool Initialize ()=0; - - /** - * Used to store thread invocation arguments. These are passed along - * to @ref FastOS_Runnable::Run() when the thread is dispatched. - */ - void *_startArg; - - /** - * Create an operating system thread. In most cases you would want - * to create threads using @ref FastOS_ThreadPool::NewThread() instead. - * @param pool The threadpool which is about to contain the new thread. - * @return A new @ref FastOS_Thread or nullptr on failure. - */ - static FastOS_ThreadInterface *CreateThread(FastOS_ThreadPool *pool); - - /** - * Break flag. If true, the thread should exit. - */ - std::atomic<bool> _breakFlag; - - /** - * Is this thread active or free in the threadpool? - */ - bool _active; - - /** - * Is the thread running? This is used by @ref Join(), to wait for threads - * to finish. - */ - std::mutex _runningMutex; - std::condition_variable _runningCond; - bool _runningFlag; - -public: - /** - * Constructor. Resets internal attributes. - */ - FastOS_ThreadInterface (FastOS_ThreadPool *pool) - : _dispatchedMutex(), - _dispatchedCond(), - _next(nullptr), - _prev(nullptr), - _owner(nullptr), - _pool(pool), - _startArg(nullptr), - _breakFlag(false), - _active(false), - _runningMutex(), - _runningCond(), - _runningFlag(false) - { - } - - /** - * Destructor. - */ - virtual ~FastOS_ThreadInterface () {} - - /** - * Instruct a thread to exit. This could be used in conjunction with - * @ref GetBreakFlag() in a worker thread, to have cooperative thread - * termination. When a threadpool closes, all threads in the pool will - * have their break flag set. - */ - void SetBreakFlag (); - - /** - * Return the status of this thread's break flag. If the break flag - * is set, someone wants the thread to terminate. It is up to the - * implementor of the thread to decide whether the break flag - * should be used. - * - * In scenarios where a worker thread loops "forever" waiting for - * new jobs, the break flag should be polled in order to eventually - * exit from the loop and terminate the thread. - * - * In scenarios where a worker thread performs a task which - * always should run to completion, the break flag could be ignored - * as the thread sooner or later will terminate. - * - * When a threadpool is closed, the break flag is set on all - * threads in the pool. If a thread loops forever and chooses to - * ignore the break flag, a @ref FastOS_ThreadPool::Close() will - * never finish. (see @ref SetBreakFlag) - */ - bool GetBreakFlag () const - { - return _breakFlag.load(std::memory_order_relaxed); - } - - /** - * Wait for a thread to finish. - */ - void Join (); - - /** - * Returns the id of this thread. - */ - virtual FastOS_ThreadId GetThreadId () const noexcept = 0; -}; - - -/** - * This class gives a generic interface for invoking new threads with an object. - * - * The thread object should inherit this interface (class), and implement - * the @ref Run() method. When @ref FastOS_ThreadPool::NewThread() is - * called, the @ref Run() method of the passed instance will be invoked. - * - * Arguments could be supplied via @ref FastOS_ThreadPool::NewThread(), but - * it is also possible to supply arguments to the new thread through the - * worker thread object constructor or some other attribute-setting method - * prior to creating the thread. Choose whichever method works best for you. - * - * Example: - * @code - * // Arguments passed to the new thread. - * struct MyThreadArgs - * { - * int _something; - * char _tenChars[10]; - * }; - * - * class MyWorkerThread : public FastOS_Runnable - * { - * public: - * - * // Delete this instance upon completion - * virtual bool DeleteOnCompletion() const { return true; } - * - * virtual void Run (FastOS_ThreadInterface *thread, void *arguments) - * { - * MyThreadArgs *args = static_cast<MyThreadArgs *>(arguments); - * - * // Do some computation... - * Foo(args->_something); - * - * for(int i=0; i<30000; i++) - * { - * ... - * ... - * - * if(thread->GetBreakFlag()) - * break; - * ... - * ... - * - * } - * - * // Thread terminates... - * } - * }; - * - * - * // Example on how to create a thread using the above classes. - * void SomeClass::SomeMethod (FastOS_ThreadPool *pool) - * { - * MyWorkerThread *workerThread = new MyWorkerThread(); - * static MyThreadArgs arguments; - * - * arguments._something = 123456; - * - * // the workerThread instance will be deleted when Run completes - * // see the DeleteOnCompletion doc - * pool->NewThread(workerThread, &arguments); - * } - * @endcode - */ -class FastOS_Runnable -{ -private: - friend class FastOS_ThreadInterface; - std::atomic<FastOS_ThreadInterface*> _thread; - -public: - FastOS_Runnable(const FastOS_Runnable&) = delete; - FastOS_Runnable& operator=(const FastOS_Runnable&) = delete; - FastOS_Runnable(); - virtual ~FastOS_Runnable(); - - /** - * The DeleteOnCompletion method should be overridden to return true - * if the runnable instance should be deleted when run completes - * - * @author Nils Sandoy - * @return true iff this runnable instance should be deleted on completion - */ - virtual bool DeleteOnCompletion() const { return false; } - - /** - * When an object implementing interface @ref FastOS_Runnable is used to - * create a thread, starting the thread causes the object's @ref Run() - * method to be called in that separately executing thread. The thread - * terminates when @ref Run() returns. - * @param thisThread A thread object. - * @param arguments Supplied to @ref FastOS_ThreadPool::NewThread - */ - virtual void Run(FastOS_ThreadInterface *thisThread, void *arguments)=0; - - FastOS_ThreadInterface *GetThread() noexcept { return _thread.load(std::memory_order_acquire); } - const FastOS_ThreadInterface *GetThread() const noexcept { return _thread.load(std::memory_order_acquire); } - bool HasThread() const noexcept { return GetThread() != nullptr; } -}; - -#include <vespa/fastos/unix_thread.h> -typedef FastOS_UNIX_Thread FASTOS_PREFIX(Thread); - diff --git a/fastos/src/vespa/fastos/types.h b/fastos/src/vespa/fastos/types.h deleted file mode 100644 index 69dd3e5231c..00000000000 --- a/fastos/src/vespa/fastos/types.h +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#define FASTOS_PREFIX(a) FastOS_##a - -// New macros to support the new gcc visibility features. -#define VESPA_DLL_EXPORT __attribute__ ((visibility("default"))) -#define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden"))) diff --git a/fastos/src/vespa/fastos/unix_thread.cpp b/fastos/src/vespa/fastos/unix_thread.cpp deleted file mode 100644 index 621505b7e02..00000000000 --- a/fastos/src/vespa/fastos/unix_thread.cpp +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "thread.h" - -bool FastOS_UNIX_Thread::Initialize () -{ - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - _handleValid = (0 == pthread_create(&_handle, &attr, FastOS_ThreadHook, this)); - pthread_attr_destroy(&attr); - - return _handleValid; -} - -FastOS_UNIX_Thread::~FastOS_UNIX_Thread() -{ - if (!_handleValid) return; - - void *value = nullptr; - pthread_join(_handle, &value); -} - -FastOS_ThreadId FastOS_UNIX_Thread::GetThreadId () const noexcept -{ - return _handle; -} - -FastOS_ThreadId FastOS_UNIX_Thread::GetCurrentThreadId () -{ - return pthread_self(); -} - -bool FastOS_UNIX_Thread::CompareThreadIds (FastOS_ThreadId a, FastOS_ThreadId b) -{ - return (pthread_equal(a, b) != 0); -} diff --git a/fastos/src/vespa/fastos/unix_thread.h b/fastos/src/vespa/fastos/unix_thread.h deleted file mode 100644 index c3c757e3fd9..00000000000 --- a/fastos/src/vespa/fastos/unix_thread.h +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** -****************************************************************************** -* @author Oivind H. Danielsen -* @date Creation date: 2000-02-02 -* @file -* Class definition for FastOS_UNIX_Thread -*****************************************************************************/ - -#pragma once - -#include "thread.h" - -class FastOS_UNIX_Thread : public FastOS_ThreadInterface -{ -protected: - pthread_t _handle; - bool _handleValid; - - bool Initialize () override; -public: - FastOS_UNIX_Thread(const FastOS_UNIX_Thread &) = delete; - FastOS_UNIX_Thread& operator=(const FastOS_UNIX_Thread &) = delete; - FastOS_UNIX_Thread(FastOS_ThreadPool *pool) - : FastOS_ThreadInterface(pool), - _handle(), - _handleValid(false) - {} - - ~FastOS_UNIX_Thread() override; - - FastOS_ThreadId GetThreadId () const noexcept override; - static bool CompareThreadIds (FastOS_ThreadId a, FastOS_ThreadId b); - static FastOS_ThreadId GetCurrentThreadId (); -}; - - diff --git a/fbench/CMakeLists.txt b/fbench/CMakeLists.txt index ff287d221ec..3f1d78a66a0 100644 --- a/fbench/CMakeLists.txt +++ b/fbench/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalib APPS diff --git a/fbench/src/httpclient/CMakeLists.txt b/fbench/src/httpclient/CMakeLists.txt index 163a68f9c98..2d5e3b32437 100644 --- a/fbench/src/httpclient/CMakeLists.txt +++ b/fbench/src/httpclient/CMakeLists.txt @@ -4,5 +4,4 @@ vespa_add_library(fbench_httpclient STATIC httpclient.cpp DEPENDS fbench_util - fastos ) diff --git a/fbench/src/test/CMakeLists.txt b/fbench/src/test/CMakeLists.txt index d13b6b82a81..c81d818ed06 100644 --- a/fbench/src/test/CMakeLists.txt +++ b/fbench/src/test/CMakeLists.txt @@ -26,6 +26,5 @@ vespa_add_executable(fbench_clientstatus_app TEST clientstatus.cpp DEPENDS fbench_util - fastos ) vespa_add_test(NAME fbench_clientstatus_app COMMAND fbench_clientstatus_app) diff --git a/fileacquirer/CMakeLists.txt b/fileacquirer/CMakeLists.txt index 13150f58ba3..cc18dc2bd84 100644 --- a/fileacquirer/CMakeLists.txt +++ b/fileacquirer/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib config_cloudconfig diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 9732a8d5ce9..a7cf41f2462 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -46,6 +46,16 @@ public class Flags { private static volatile TreeMap<FlagId, FlagDefinition> flags = new TreeMap<>(); + public static final UnboundBooleanFlag DROP_CACHES = defineFeatureFlag( + "drop-caches", false, + List.of("hakonhall", "baldersheim"), "2023-03-06", "2023-04-05", + "Drop caches on tenant hosts", + "Takes effect on next tick", + ZONE_ID, + // The application ID is the exclusive application ID associated with the host, + // if any, or otherwise hosted-vespa:tenant-host:default. + APPLICATION_ID); + public static final UnboundDoubleFlag DEFAULT_TERM_WISE_LIMIT = defineDoubleFlag( "default-term-wise-limit", 1.0, List.of("baldersheim"), "2020-12-02", "2023-12-31", @@ -203,9 +213,10 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + // TODO: Move to a permanent flag public static final UnboundListFlag<String> ALLOWED_ATHENZ_PROXY_IDENTITIES = defineListFlag( "allowed-athenz-proxy-identities", List.of(), String.class, - List.of("bjorncs", "tokle"), "2021-02-10", "2023-03-01", + List.of("bjorncs", "tokle"), "2021-02-10", "2023-05-01", "Allowed Athenz proxy identities", "takes effect at redeployment"); @@ -281,13 +292,6 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag NOTIFICATION_DISPATCH_FLAG = defineFeatureFlag( - "dispatch-notifications", false, - List.of("enygaard"), "2022-05-02", "2023-03-01", - "Whether we should send notification for a given tenant", - "Takes effect immediately", - TENANT_ID); - public static final UnboundBooleanFlag ENABLE_PROXY_PROTOCOL_MIXED_MODE = defineFeatureFlag( "enable-proxy-protocol-mixed-mode", true, List.of("tokle"), "2022-05-09", "2023-03-31", @@ -338,13 +342,6 @@ public class Flags { "Takes effect immediately", CONSOLE_USER_EMAIL); - public static final UnboundBooleanFlag USE_WIREGUARD_ON_CONFIGSERVERS = defineFeatureFlag( - "use-wireguard-on-configservers", false, - List.of("andreer", "gjoranv"), "2022-09-28", "2023-04-01", - "Set up a WireGuard endpoint on config servers", - "Takes effect on configserver restart", - HOSTNAME); - public static final UnboundStringFlag CORE_ENCRYPTION_PUBLIC_KEY_ID = defineStringFlag( "core-encryption-public-key-id", "", List.of("vekterli"), "2022-11-03", "2023-05-01", @@ -352,6 +349,20 @@ public class Flags { "Takes effect on the next tick.", ZONE_ID, NODE_TYPE, HOSTNAME); + public static final UnboundBooleanFlag ENABLE_GLOBAL_PHASE = defineFeatureFlag( + "enable-global-phase", false, + List.of("arnej", "bjorncs"), "2023-02-28", "2024-01-10", + "Enable global phase ranking", + "Takes effect at redeployment", + APPLICATION_ID); + + public static final UnboundBooleanFlag VESPA_ATHENZ_PROVIDER = defineFeatureFlag( + "vespa-athenz-provider", false, + List.of("mortent"), "2023-02-22", "2023-05-01", + "Enable athenz provider in public systems", + "Takes effect on next config server container start", + ZONE_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index 384fa7a4177..9aa6ef414fc 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -85,7 +85,7 @@ public class PermanentFlags { "host-flavor", "", "Specifies the Vespa flavor name that the hosts of the matching nodes should have.", "Takes effect on next deployment (including internal redeployment).", - APPLICATION_ID, CLUSTER_TYPE); + APPLICATION_ID, CLUSTER_TYPE, CLUSTER_ID); public static final UnboundBooleanFlag SKIP_MAINTENANCE_DEPLOYMENT = defineFeatureFlag( "node-repository-skip-maintenance-deployment", false, @@ -322,14 +322,20 @@ public class PermanentFlags { APPLICATION_ID); public static final UnboundLongFlag CONFIG_SERVER_SESSION_EXPIRY_TIME = defineLongFlag( - // TODO: Lower to 3600, which is default session expiry time - "config-server-session-expiry-time", 3600 * 2, + "config-server-session-expiry-time", 3600, "Expiry time in seconds for remote sessions (session in ZooKeeper). Default should be equal to session lifetime, " + "but can be lowered if there are incidents/bugs where one needs to delete sessions", "Takes effect immediately", ZONE_ID ); + public static final UnboundBooleanFlag NOTIFICATION_DISPATCH_FLAG = defineFeatureFlag( + "dispatch-notifications", true, + "Whether we should send notification for a given tenant", + "Takes effect immediately", + TENANT_ID); + + private PermanentFlags() {} private static UnboundBooleanFlag defineFeatureFlag( diff --git a/fnet/CMakeLists.txt b/fnet/CMakeLists.txt index eeef7f63876..6d8836817e0 100644 --- a/fnet/CMakeLists.txt +++ b/fnet/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib diff --git a/fnet/INSTALL b/fnet/INSTALL deleted file mode 100644 index 9bb6c6152a4..00000000000 --- a/fnet/INSTALL +++ /dev/null @@ -1,22 +0,0 @@ -********************************* -* Compiling and Installing fnet * -********************************* - -'fnet' uses 'FastOS'. -In the following instructions, let %FASTOS_DIR% denote the install -directory for FastOS. A reasonable install directory would be: - %FASTOS_DIR% = '/usr/fastsearch/fastos' - -Install FastOS: -- checkout the fastos CVS module -- go to fastos/src/fastos -- ./configure --install-dir %FASTOS_DIR% [<config parameters>] - (run ./configure --help for help) -- make install - -Install fnet: -- checkout the fnet CVS module -- go to fnet/src -- ./configure --fastos-dir %FASTOS_DIR% [<config parameters>] - (run ./configure --fastos-dir %FASTOS_DIR% --help for help) -- make install diff --git a/fnet/README b/fnet/README deleted file mode 100644 index dab36e5b120..00000000000 --- a/fnet/README +++ /dev/null @@ -1,26 +0,0 @@ -This cvs module contains the code for FNET and FRT -FNET currently stands for 'FuNET' -FRT currently stands for 'FNET Remote Tools' - -FNET is a multi-threaded object-oriented networking library based on -FastOS. FRT implements a proprietary RPC protocol on top of FNET. - -For how to compile and install, read the INSTALL file. -For release information, read the RELEASEINFO file. - -The maintainer of this module is [havardpe@yahoo-inc.com] - -Notes from the maintainer: - -This library is work in progress. This means that some APIs may change -in new versions. I will try to keep these changes to a minimum and -also document them in the RELEASEINFO file. - -No extra effort has been added to hide things from the users of this -library. This means that you can use components like the scheduler to -implement a scheduler thread that has nothing to do with -networking. It also means that you have to expect to handle more API -changes if you are using classes not intended for direct use. It also -means that you will have access to methods you were never meant to -invoke. If this becomes a problem, I will start making things private -and the classes more 'friendly'. diff --git a/fnet/src/examples/frt/rpc/echo_client.cpp b/fnet/src/examples/frt/rpc/echo_client.cpp index 0176337c466..869a010bfda 100644 --- a/fnet/src/examples/frt/rpc/echo_client.cpp +++ b/fnet/src/examples/frt/rpc/echo_client.cpp @@ -85,8 +85,8 @@ EchoClient::main(int argc, char **argv) } else { printf("Return values != parameters.\n"); } - req->SubRef(); - target->SubRef(); + req->internal_subref(); + target->internal_subref(); return 0; } diff --git a/fnet/src/examples/frt/rpc/rpc_callback_client.cpp b/fnet/src/examples/frt/rpc/rpc_callback_client.cpp index b41c40ba29d..c52d48f24eb 100644 --- a/fnet/src/examples/frt/rpc/rpc_callback_client.cpp +++ b/fnet/src/examples/frt/rpc/rpc_callback_client.cpp @@ -102,8 +102,8 @@ MyApp::main(int argc, char **argv) ok = false; } - req->SubRef(); - target->SubRef(); + req->internal_subref(); + target->internal_subref(); return ok ? 0 : 1; } diff --git a/fnet/src/examples/frt/rpc/rpc_callback_server.cpp b/fnet/src/examples/frt/rpc/rpc_callback_server.cpp index 37a819c6ee7..5c21f73da7f 100644 --- a/fnet/src/examples/frt/rpc/rpc_callback_server.cpp +++ b/fnet/src/examples/frt/rpc/rpc_callback_server.cpp @@ -27,7 +27,7 @@ void do_callback(FRT_RPCRequest *req) { cb->GetErrorCode(), cb->GetErrorMessage()); } - cb->SubRef(); + cb->internal_subref(); req->Return(); } diff --git a/fnet/src/examples/frt/rpc/rpc_client.cpp b/fnet/src/examples/frt/rpc/rpc_client.cpp index acb20a880c9..55168a2acba 100644 --- a/fnet/src/examples/frt/rpc/rpc_client.cpp +++ b/fnet/src/examples/frt/rpc/rpc_client.cpp @@ -49,7 +49,7 @@ RPCClient::main(int argc, char **argv) } fprintf(stdout, "\nTesting addFloat method\n"); - req->SubRef(); + req->internal_subref(); req = supervisor.AllocRPCRequest(); req->SetMethodName("addFloat"); req->GetParams()->AddFloat(float1); @@ -65,7 +65,7 @@ RPCClient::main(int argc, char **argv) } fprintf(stdout, "\nTesting addDouble method\n"); - req->SubRef(); + req->internal_subref(); req = supervisor.AllocRPCRequest(); req->SetMethodName("addDouble"); req->GetParams()->AddDouble(double1); @@ -80,8 +80,8 @@ RPCClient::main(int argc, char **argv) req->GetErrorMessage()); } - req->SubRef(); - target->SubRef(); + req->internal_subref(); + target->internal_subref(); return 0; } diff --git a/fnet/src/examples/frt/rpc/rpc_info.cpp b/fnet/src/examples/frt/rpc/rpc_info.cpp index 9734342a24e..ab534a254c2 100644 --- a/fnet/src/examples/frt/rpc/rpc_info.cpp +++ b/fnet/src/examples/frt/rpc/rpc_info.cpp @@ -15,16 +15,16 @@ public: void GetReq(FRT_RPCRequest **req, FRT_Supervisor *supervisor) { if ((*req) != nullptr) - (*req)->SubRef(); + (*req)->internal_subref(); (*req) = supervisor->AllocRPCRequest(); } void FreeReqs(FRT_RPCRequest *r1, FRT_RPCRequest *r2) { if (r1 != nullptr) - r1->SubRef(); + r1->internal_subref(); if (r2 != nullptr) - r2->SubRef(); + r2->internal_subref(); } void DumpMethodInfo(const char *indent, FRT_RPCRequest *info, @@ -130,7 +130,7 @@ RPCInfo::main(int argc, char **argv) m_list->GetErrorMessage()); } FreeReqs(m_list, info); - target->SubRef(); + target->internal_subref(); return 0; } diff --git a/fnet/src/examples/frt/rpc/rpc_invoke.cpp b/fnet/src/examples/frt/rpc/rpc_invoke.cpp index 9f3e90f469a..d56847098d8 100644 --- a/fnet/src/examples/frt/rpc/rpc_invoke.cpp +++ b/fnet/src/examples/frt/rpc/rpc_invoke.cpp @@ -113,8 +113,8 @@ RPCClient::run(int argc, char **argv) retCode = 3; } } - req->SubRef(); - target->SubRef(); + req->internal_subref(); + target->internal_subref(); return retCode; } diff --git a/fnet/src/examples/ping/pingclient.cpp b/fnet/src/examples/ping/pingclient.cpp index 9b32e40ac83..b59df31607a 100644 --- a/fnet/src/examples/ping/pingclient.cpp +++ b/fnet/src/examples/ping/pingclient.cpp @@ -6,7 +6,6 @@ #include <vespa/fnet/connection.h> #include <examples/ping/packets.h> #include <vespa/vespalib/util/signalhandler.h> -#include <vespa/fastos/thread.h> #include <vespa/log/log.h> LOG_SETUP("pingclient"); @@ -28,7 +27,6 @@ PingClient::main(int argc, char **argv) } FNET_PacketQueue queue; - FastOS_ThreadPool pool; PingPacketFactory factory; FNET_SimplePacketStreamer streamer(&factory); FNET_Transport transport; @@ -39,7 +37,7 @@ PingClient::main(int argc, char **argv) if (argc == 3) { timeout_ms = atof(argv[2]) * 1000; } - transport.Start(&pool); + transport.Start(); uint32_t channelCnt = 0; for (uint32_t i = 0; i < 10; i++) { @@ -88,9 +86,8 @@ PingClient::main(int argc, char **argv) packet->Free(); } if (conn != nullptr) - conn->SubRef(); + conn->internal_subref(); transport.ShutDown(true); - pool.Close(); return 0; } diff --git a/fnet/src/examples/ping/pingserver.cpp b/fnet/src/examples/ping/pingserver.cpp index fb5b12b66c0..79a67cd18a7 100644 --- a/fnet/src/examples/ping/pingserver.cpp +++ b/fnet/src/examples/ping/pingserver.cpp @@ -51,7 +51,7 @@ PingServer::main(int argc, char **argv) FNET_Connector *listener = transport.Listen(argv[1], &streamer, this); if (listener != nullptr) - listener->SubRef(); + listener->internal_subref(); FNET_SignalShutDown ssd(transport); transport.Main(); diff --git a/fnet/src/examples/timeout/timeout.cpp b/fnet/src/examples/timeout/timeout.cpp index 41de852d48c..e0830c7cde1 100644 --- a/fnet/src/examples/timeout/timeout.cpp +++ b/fnet/src/examples/timeout/timeout.cpp @@ -5,7 +5,6 @@ #include <vespa/fnet/packetqueue.h> #include <vespa/fnet/controlpacket.h> #include <vespa/vespalib/util/signalhandler.h> -#include <vespa/fastos/thread.h> #include <vespa/vespalib/util/time.h> #include <thread> @@ -54,10 +53,9 @@ MyApp::main(int, char **) ms_double ms; clock::time_point t; FNET_PacketQueue queue; - FastOS_ThreadPool pool; FNET_Transport transport; Timeout timeout(transport.GetScheduler(), &queue); - transport.Start(&pool); + transport.Start(); // stable-state operation std::this_thread::sleep_for(100ms); @@ -90,7 +88,6 @@ MyApp::main(int, char **) fprintf(stderr, "time since timeout was scheduled: %f ms\n", ms.count()); transport.ShutDown(true); - pool.Close(); return 0; } diff --git a/fnet/src/tests/connect/connect_test.cpp b/fnet/src/tests/connect/connect_test.cpp index 681c3b7676a..d635fea6f94 100644 --- a/fnet/src/tests/connect/connect_test.cpp +++ b/fnet/src/tests/connect/connect_test.cpp @@ -86,28 +86,24 @@ struct BlockingCryptoEngine : public CryptoEngine { //----------------------------------------------------------------------------- -struct TransportFixture : FNET_IPacketHandler, FNET_IConnectionCleanupHandler { +struct TransportFixture : FNET_IPacketHandler { FNET_SimplePacketStreamer streamer; - FastOS_ThreadPool pool; FNET_Transport transport; Gate conn_lost; - Gate conn_deleted; - TransportFixture() : streamer(nullptr), pool(), transport(), - conn_lost(), conn_deleted() - { - transport.Start(&pool); + TransportFixture() : streamer(nullptr), transport(), conn_lost() { + transport.Start(); } TransportFixture(AsyncResolver::HostResolver::SP host_resolver) - : streamer(nullptr), pool(), transport(fnet::TransportConfig().resolver(make_resolver(std::move(host_resolver)))), - conn_lost(), conn_deleted() + : streamer(nullptr), transport(fnet::TransportConfig().resolver(make_resolver(std::move(host_resolver)))), + conn_lost() { - transport.Start(&pool); + transport.Start(); } TransportFixture(CryptoEngine::SP crypto) - : streamer(nullptr), pool(), transport(fnet::TransportConfig().crypto(std::move(crypto))), - conn_lost(), conn_deleted() + : streamer(nullptr), transport(fnet::TransportConfig().crypto(std::move(crypto))), + conn_lost() { - transport.Start(&pool); + transport.Start(); } HP_RetCode HandlePacket(FNET_Packet *packet, FNET_Context) override { ASSERT_TRUE(packet->GetCommand() == FNET_ControlPacket::FNET_CMD_CHANNEL_LOST); @@ -115,26 +111,41 @@ struct TransportFixture : FNET_IPacketHandler, FNET_IConnectionCleanupHandler { packet->Free(); return FNET_FREE_CHANNEL; } - void Cleanup(FNET_Connection *) override { conn_deleted.countDown(); } FNET_Connection *connect(const vespalib::string &spec) { FNET_Connection *conn = transport.Connect(spec.c_str(), &streamer); ASSERT_TRUE(conn != nullptr); if (conn->OpenChannel(this, FNET_Context()) == nullptr) { conn_lost.countDown(); } - conn->SetCleanupHandler(this); return conn; } ~TransportFixture() override { transport.ShutDown(true); - pool.Close(); } }; //----------------------------------------------------------------------------- -TEST_MT_FFF("require that normal connect works", 2, - ServerSocket("tcp/0"), TransportFixture(), TimeBomb(60)) +struct ConnCheck { + uint64_t target; + ConnCheck() : target(FNET_Connection::get_num_connections()) { + EXPECT_EQUAL(target, uint64_t(0)); + } + bool at_target() const { return (FNET_Connection::get_num_connections() == target); }; + bool await(duration max_wait) const { + auto until = saturated_add(steady_clock::now(), max_wait); + while (!at_target() && steady_clock::now() < until) { + std::this_thread::sleep_for(1ms); + } + return at_target(); + } + void await() const { + ASSERT_TRUE(await(3600s)); + } +}; + +TEST_MT_FFFF("require that normal connect works", 2, + ServerSocket("tcp/0"), TransportFixture(), ConnCheck(), TimeBomb(60)) { if (thread_id == 0) { SocketHandle socket = f1.accept(); @@ -146,23 +157,23 @@ TEST_MT_FFF("require that normal connect works", 2, TEST_BARRIER(); conn->Owner()->Close(conn); f2.conn_lost.await(); - EXPECT_TRUE(!f2.conn_deleted.await(short_time)); - conn->SubRef(); - f2.conn_deleted.await(); + EXPECT_TRUE(!f3.await(short_time)); + conn->internal_subref(); + f3.await(); } } -TEST_FF("require that bogus connect fail asynchronously", TransportFixture(), TimeBomb(60)) { +TEST_FFF("require that bogus connect fail asynchronously", TransportFixture(), ConnCheck(), TimeBomb(60)) { FNET_Connection *conn = f1.connect("invalid"); f1.conn_lost.await(); - EXPECT_TRUE(!f1.conn_deleted.await(short_time)); - conn->SubRef(); - f1.conn_deleted.await(); + EXPECT_TRUE(!f2.await(short_time)); + conn->internal_subref(); + f2.await(); } -TEST_MT_FFFF("require that async close can be called before async resolve completes", 2, - ServerSocket("tcp/0"), std::shared_ptr<BlockingHostResolver>(new BlockingHostResolver()), - TransportFixture(f2), TimeBomb(60)) +TEST_MT_FFFFF("require that async close can be called before async resolve completes", 2, + ServerSocket("tcp/0"), std::shared_ptr<BlockingHostResolver>(new BlockingHostResolver()), + TransportFixture(f2), ConnCheck(), TimeBomb(60)) { if (thread_id == 0) { SocketHandle socket = f1.accept(); @@ -174,16 +185,16 @@ TEST_MT_FFFF("require that async close can be called before async resolve comple conn->Owner()->Close(conn); f3.conn_lost.await(); f2->release_caller(); - EXPECT_TRUE(!f3.conn_deleted.await(short_time)); - conn->SubRef(); - f3.conn_deleted.await(); + EXPECT_TRUE(!f4.await(short_time)); + conn->internal_subref(); + f4.await(); f1.shutdown(); } } -TEST_MT_FFFF("require that async close during async do_handshake_work works", 2, - ServerSocket("tcp/0"), std::shared_ptr<BlockingCryptoEngine>(new BlockingCryptoEngine()), - TransportFixture(f2), TimeBomb(60)) +TEST_MT_FFFFF("require that async close during async do_handshake_work works", 2, + ServerSocket("tcp/0"), std::shared_ptr<BlockingCryptoEngine>(new BlockingCryptoEngine()), + TransportFixture(f2), ConnCheck(), TimeBomb(60)) { if (thread_id == 0) { SocketHandle socket = f1.accept(); @@ -198,10 +209,10 @@ TEST_MT_FFFF("require that async close during async do_handshake_work works", 2, f3.conn_lost.await(); TEST_BARRIER(); // #1 // verify that pending work keeps relevant objects alive - EXPECT_TRUE(!f3.conn_deleted.await(short_time)); + EXPECT_TRUE(!f4.await(short_time)); EXPECT_TRUE(!f2->handshake_socket_deleted.await(short_time)); f2->handshake_work_exit.countDown(); - f3.conn_deleted.await(); + f4.await(); f2->handshake_socket_deleted.await(); } } diff --git a/fnet/src/tests/connection_spread/connection_spread_test.cpp b/fnet/src/tests/connection_spread/connection_spread_test.cpp index d65e4fb70fe..5908e6a4982 100644 --- a/fnet/src/tests/connection_spread/connection_spread_test.cpp +++ b/fnet/src/tests/connection_spread/connection_spread_test.cpp @@ -6,7 +6,6 @@ #include <vespa/fnet/ipacketstreamer.h> #include <vespa/fnet/connector.h> #include <vespa/fnet/connection.h> -#include <vespa/fastos/thread.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/stringfmt.h> #include <thread> @@ -28,13 +27,12 @@ struct DummyStreamer : FNET_IPacketStreamer { struct Fixture { DummyStreamer streamer; DummyAdapter adapter; - FastOS_ThreadPool thread_pool; FNET_Transport client; FNET_Transport server; - Fixture() : streamer(), adapter(), thread_pool(), client(8), server(8) + Fixture() : streamer(), adapter(), client(8), server(8) { - ASSERT_TRUE(client.Start(&thread_pool)); - ASSERT_TRUE(server.Start(&thread_pool)); + ASSERT_TRUE(client.Start()); + ASSERT_TRUE(server.Start()); } void wait_for_components(size_t client_cnt, size_t server_cnt) { bool ok = false; @@ -49,7 +47,6 @@ struct Fixture { ~Fixture() { server.ShutDown(true); client.ShutDown(true); - thread_pool.Close(); } }; @@ -87,9 +84,9 @@ TEST_F("require that connections are spread among transport threads", Fixture) f1.wait_for_components(256, 257); check_threads(f1.client, 8, "client"); check_threads(f1.server, 8, "server"); - listener->SubRef(); + listener->internal_subref(); for (FNET_Connection *conn: connections) { - conn->SubRef(); + conn->internal_subref(); } } diff --git a/fnet/src/tests/frt/detach_supervisor/detach_supervisor_test.cpp b/fnet/src/tests/frt/detach_supervisor/detach_supervisor_test.cpp index 716c433ff61..c5ca6dc6ce9 100644 --- a/fnet/src/tests/frt/detach_supervisor/detach_supervisor_test.cpp +++ b/fnet/src/tests/frt/detach_supervisor/detach_supervisor_test.cpp @@ -10,7 +10,6 @@ #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/time.h> -#include <vespa/fastos/thread.h> #include <thread> using namespace vespalib; @@ -19,14 +18,12 @@ using vespalib::make_string_short::fmt; CryptoEngine::SP null_crypto = std::make_shared<NullCryptoEngine>(); struct BasicFixture { - FastOS_ThreadPool thread_pool; FNET_Transport transport; - BasicFixture() : thread_pool(), transport(fnet::TransportConfig(4).crypto(null_crypto)) { - ASSERT_TRUE(transport.Start(&thread_pool)); + BasicFixture() : transport(fnet::TransportConfig(4).crypto(null_crypto)) { + ASSERT_TRUE(transport.Start()); } ~BasicFixture() { transport.ShutDown(true); - thread_pool.Close(); } }; @@ -39,7 +36,7 @@ struct RpcFixture : FRT_Invokable { } ~RpcFixture() { if (back_conn.load() != nullptr) { - back_conn.load()->SubRef(); + back_conn.load()->internal_subref(); } } uint32_t port() const { return orb.GetListenPort(); } @@ -64,7 +61,7 @@ struct RpcFixture : FRT_Invokable { ASSERT_TRUE(back_conn.load() == nullptr); back_conn.store(req->GetConnection()); ASSERT_TRUE(back_conn.load() != nullptr); - back_conn.load()->AddRef(); + back_conn.load()->internal_addref(); } FRT_Target *meta_connect(uint32_t port) { auto *target = orb.Get2WayTarget(fmt("tcp/localhost:%u", port).c_str()); @@ -72,7 +69,7 @@ struct RpcFixture : FRT_Invokable { req->SetMethodName("connect"); target->InvokeSync(req, 300.0); ASSERT_TRUE(req->CheckReturnTypes("")); - req->SubRef(); + req->internal_subref(); return target; }; static int check_result(FRT_RPCRequest *req, uint64_t expect) { @@ -84,7 +81,7 @@ struct RpcFixture : FRT_Invokable { ASSERT_EQUAL(ret, expect); ++num_ok; } - req->SubRef(); + req->internal_subref(); return num_ok; } static int verify_rpc(FNET_Connection *conn) { @@ -104,7 +101,7 @@ struct RpcFixture : FRT_Invokable { int verify_rpc(FRT_Target *target, uint32_t port) { auto *my_target = connect(port); int num_ok = verify_rpc(target) + verify_rpc(my_target) + verify_rpc(back_conn.load()); - my_target->SubRef(); + my_target->internal_subref(); return num_ok; } }; @@ -138,8 +135,8 @@ TEST_MT_FFFFF("require that supervisor can be detached from transport", 4, Basic EXPECT_EQUAL(RpcFixture::verify_rpc(target), 0); // outgoing 2way target should be closed EXPECT_EQUAL(RpcFixture::verify_rpc(client_target), 1); // pure client target should not be closed TEST_BARRIER(); // #5 - target->SubRef(); - client_target->SubRef(); + target->internal_subref(); + client_target->internal_subref(); } else if (thread_id == 1) { // server 2 (talks to client 2) auto self = std::make_unique<RpcFixture>(f1); f3 = self->port(); @@ -149,7 +146,7 @@ TEST_MT_FFFFF("require that supervisor can be detached from transport", 4, Basic TEST_BARRIER(); // #3 TEST_BARRIER(); // #4 TEST_BARRIER(); // #5 - target->SubRef(); + target->internal_subref(); } else if (thread_id == 2) { // client 1 (talks to server 1) auto self = std::make_unique<RpcFixture>(f1); f4 = self->port(); @@ -168,7 +165,7 @@ TEST_MT_FFFFF("require that supervisor can be detached from transport", 4, Basic TEST_BARRIER(); // #4 EXPECT_EQUAL(self->verify_rpc(target, f2), 0); TEST_BARRIER(); // #5 - target->SubRef(); + target->internal_subref(); } else { // client 2 (talks to server 2) ASSERT_EQUAL(thread_id, 3u); auto self = std::make_unique<RpcFixture>(f1); @@ -182,7 +179,7 @@ TEST_MT_FFFFF("require that supervisor can be detached from transport", 4, Basic TEST_BARRIER(); // #4 EXPECT_EQUAL(self->verify_rpc(target, f3), 3); TEST_BARRIER(); // #5 - target->SubRef(); + target->internal_subref(); } } diff --git a/fnet/src/tests/frt/method_pt/method_pt.cpp b/fnet/src/tests/frt/method_pt/method_pt.cpp index 2ac706369ae..d6c42ef7790 100644 --- a/fnet/src/tests/frt/method_pt/method_pt.cpp +++ b/fnet/src/tests/frt/method_pt/method_pt.cpp @@ -174,7 +174,7 @@ void finiTest() { delete _complexHandler; delete _mediumHandler; delete _simpleHandler; - _target->SubRef(); + _target->internal_subref(); _server.reset(); } @@ -187,7 +187,7 @@ TEST("method pt") { //-------------------------------- MEDIUM - req->SubRef(); + req->internal_subref(); req = FRT_Supervisor::AllocRPCRequest(); req->SetMethodName("mediumMethod"); _target->InvokeSync(req, 60.0); @@ -195,7 +195,7 @@ TEST("method pt") { //-------------------------------- COMPLEX - req->SubRef(); + req->internal_subref(); req = FRT_Supervisor::AllocRPCRequest(); req->SetMethodName("complexMethod"); _target->InvokeSync(req, 60.0); @@ -213,7 +213,7 @@ TEST("method pt") { fprintf(stderr, "Object inheritance NOT ok for method handlers\n"); } - req->SubRef(); + req->internal_subref(); } //------------------------------------------------------------- diff --git a/fnet/src/tests/frt/parallel_rpc/parallel_rpc_test.cpp b/fnet/src/tests/frt/parallel_rpc/parallel_rpc_test.cpp index 624f5a73ae6..74f69541d8b 100644 --- a/fnet/src/tests/frt/parallel_rpc/parallel_rpc_test.cpp +++ b/fnet/src/tests/frt/parallel_rpc/parallel_rpc_test.cpp @@ -4,7 +4,6 @@ #include <vespa/fnet/frt/rpcrequest.h> #include <vespa/fnet/frt/target.h> #include <vespa/fnet/transport.h> -#include <vespa/fastos/thread.h> #include <vespa/vespalib/util/benchmark_timer.h> #include <vespa/vespalib/net/crypto_engine.h> #include <vespa/vespalib/net/tls/tls_crypto_engine.h> @@ -15,13 +14,12 @@ using namespace vespalib; struct Rpc : FRT_Invokable { - FastOS_ThreadPool thread_pool; FNET_Transport transport; FRT_Supervisor orb; Rpc(CryptoEngine::SP crypto, size_t num_threads, bool drop_empty) - : thread_pool(), transport(fnet::TransportConfig(num_threads).crypto(std::move(crypto)).drop_empty_buffers(drop_empty)), orb(&transport) {} + : transport(fnet::TransportConfig(num_threads).crypto(std::move(crypto)).drop_empty_buffers(drop_empty)), orb(&transport) {} void start() { - ASSERT_TRUE(transport.Start(&thread_pool)); + ASSERT_TRUE(transport.Start()); } uint32_t listen() { ASSERT_TRUE(orb.Listen(0)); @@ -32,7 +30,6 @@ struct Rpc : FRT_Invokable { } ~Rpc() override { transport.ShutDown(true); - thread_pool.Close(); } }; @@ -128,8 +125,8 @@ void perform_test(size_t thread_id, Client &client, Result &result, bool vital = BenchmarkTimer::benchmark(invoke, invoke, 0.5); EXPECT_GREATER_EQUAL(seq, loop_cnt); result.req_per_sec[thread_id] = double(loop_cnt) / t; - req->SubRef(); - target->SubRef(); + req->internal_subref(); + target->internal_subref(); TEST_BARRIER(); if (thread_id == 0) { result.print(); diff --git a/fnet/src/tests/frt/parallel_rpc/tls_rpc_bench.cpp b/fnet/src/tests/frt/parallel_rpc/tls_rpc_bench.cpp index d15fca93c0b..812f0a57c5e 100644 --- a/fnet/src/tests/frt/parallel_rpc/tls_rpc_bench.cpp +++ b/fnet/src/tests/frt/parallel_rpc/tls_rpc_bench.cpp @@ -86,7 +86,7 @@ void benchmark_rpc(Fixture &fixture, bool reconnect) { auto invoke = [&seq, &target, &req, &fixture, reconnect](){ TT_Sample sample(req_tag); if (reconnect) { - target->SubRef(); + target->internal_subref(); target = fixture.connect(); } req = fixture.orb.AllocRPCRequest(req); @@ -101,8 +101,8 @@ void benchmark_rpc(Fixture &fixture, bool reconnect) { auto before = TimeTracer::now(); double t = BenchmarkTimer::benchmark(invoke, 5.0); auto after = TimeTracer::now(); - target->SubRef(); - req->SubRef(); + target->internal_subref(); + req->internal_subref(); auto stats = TimeTracer::extract().by_time(before, after).by_tag(req_tag.id()).get(); ASSERT_TRUE(stats.size() > 0); std::sort(stats.begin(), stats.end(), DurationCmp()); diff --git a/fnet/src/tests/frt/rpc/detach_return_invoke.cpp b/fnet/src/tests/frt/rpc/detach_return_invoke.cpp index 17c38ab6e3a..9a0f1778cb6 100644 --- a/fnet/src/tests/frt/rpc/detach_return_invoke.cpp +++ b/fnet/src/tests/frt/rpc/detach_return_invoke.cpp @@ -31,14 +31,14 @@ struct Server : public FRT_Invokable void rpc_hook(FRT_RPCRequest *req) { FNET_Connection *conn = req->GetConnection(); - conn->AddRef(); // need to keep it alive + conn->internal_addref(); // need to keep it alive req->Detach(); req->Return(); // will free request channel FRT_RPCRequest *r = orb.AllocRPCRequest(); r->SetMethodName("frt.rpc.ping"); // might re-use request channel before it is unlinked from hashmap orb.InvokeAsync(orb.GetTransport(), conn, r, 5.0, &receptor); - conn->SubRef(); // invocation will now keep the connection alive as needed + conn->internal_subref(); // invocation will now keep the connection alive as needed } }; @@ -61,11 +61,11 @@ TEST("detach return invoke") { } std::this_thread::sleep_for(10ms); } - req->SubRef(); - target->SubRef(); + req->internal_subref(); + target->internal_subref(); if (receptor.req.load() != nullptr) { EXPECT_TRUE(!receptor.req.load()->IsError()); - receptor.req.load()->SubRef(); + receptor.req.load()->internal_subref(); } EXPECT_TRUE(receptor.req.load() != nullptr); }; diff --git a/fnet/src/tests/frt/rpc/invoke.cpp b/fnet/src/tests/frt/rpc/invoke.cpp index e930c1252bf..f06c7428c22 100644 --- a/fnet/src/tests/frt/rpc/invoke.cpp +++ b/fnet/src/tests/frt/rpc/invoke.cpp @@ -65,7 +65,7 @@ public: } ~MyReq() { if (_req != nullptr) { - _req->SubRef(); + _req->internal_subref(); } } MyReq(const MyReq &rhs) = delete; @@ -331,7 +331,7 @@ public: } ~Fixture() { - _target->SubRef(); + _target->internal_subref(); } }; @@ -410,7 +410,7 @@ TEST_F("require that a bad target gives connection error", Fixture()) { { FRT_Target *bad_target = f1.make_bad_target(); bad_target->InvokeSync(req.borrow(), timeout); - bad_target->SubRef(); + bad_target->internal_subref(); } EXPECT_EQUAL(req.get().GetErrorCode(), FRTE_RPC_CONNECTION); } diff --git a/fnet/src/tests/frt/rpc/sharedblob.cpp b/fnet/src/tests/frt/rpc/sharedblob.cpp index 2ccb44d03cb..94f57a136a4 100644 --- a/fnet/src/tests/frt/rpc/sharedblob.cpp +++ b/fnet/src/tests/frt/rpc/sharedblob.cpp @@ -122,7 +122,7 @@ struct ServerSampler : public FRT_Invokable dataSet.sample(*req->GetReturn()); // server return before drop // keep request to sample return after drop - req->AddRef(); + req->internal_addref(); serverReq = req; } }; @@ -176,7 +176,7 @@ TEST("testExplicitShared") { req->GetParams()->AddSharedData(&blob); EXPECT_EQUAL(4, blob.refcnt); - req->SubRef(); + req->internal_subref(); EXPECT_EQUAL(1, blob.refcnt); } @@ -262,10 +262,10 @@ TEST("testImplicitShared") { } if (serverSampler.serverReq != 0) { - serverSampler.serverReq->SubRef(); + serverSampler.serverReq->internal_subref(); } - req->SubRef(); - target->SubRef(); + req->internal_subref(); + target->internal_subref(); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/fnet/src/tests/info/info.cpp b/fnet/src/tests/info/info.cpp index 00075cb75dd..f57f7d78b07 100644 --- a/fnet/src/tests/info/info.cpp +++ b/fnet/src/tests/info/info.cpp @@ -63,9 +63,9 @@ TEST("info") { fprintf(stderr, "FD_SETSIZE: %d\n", l[2]._intval32); fprintf(stderr, "sizeof(FRT_RPCRequest): %d\n", l[3]._intval32); - target->SubRef(); - local_info->SubRef(); - remote_info->SubRef(); + target->internal_subref(); + local_info->internal_subref(); + remote_info->internal_subref(); }; TEST("size of important objects") @@ -77,7 +77,7 @@ TEST("size of important objects") #else constexpr size_t MUTEX_SIZE = 40u; #endif - EXPECT_EQUAL(MUTEX_SIZE + sizeof(std::string) + 112u, sizeof(FNET_IOComponent)); + EXPECT_EQUAL(MUTEX_SIZE + sizeof(std::string) + 120u, sizeof(FNET_IOComponent)); EXPECT_EQUAL(32u, sizeof(FNET_Channel)); EXPECT_EQUAL(40u, sizeof(FNET_PacketQueue_NoLock)); EXPECT_EQUAL(MUTEX_SIZE + sizeof(std::string) + 416u, sizeof(FNET_Connection)); diff --git a/fnet/src/tests/printstuff/printstuff_test.cpp b/fnet/src/tests/printstuff/printstuff_test.cpp index a9621728c5a..bd76ad29405 100644 --- a/fnet/src/tests/printstuff/printstuff_test.cpp +++ b/fnet/src/tests/printstuff/printstuff_test.cpp @@ -37,7 +37,7 @@ TEST("rpc packets in a queue") { q2.QueuePacket(&req->getStash().create<FRT_RPCRequestPacket>(req, 0, false), FNET_Context()); q2.Print(); } - req->SubRef(); + req->internal_subref(); } TEST("info") { diff --git a/fnet/src/tests/sync_execute/sync_execute.cpp b/fnet/src/tests/sync_execute/sync_execute.cpp index b8fa21cf147..5d2f4097ab4 100644 --- a/fnet/src/tests/sync_execute/sync_execute.cpp +++ b/fnet/src/tests/sync_execute/sync_execute.cpp @@ -4,7 +4,6 @@ #include <vespa/vespalib/util/size_literals.h> #include <vespa/fnet/transport.h> #include <vespa/fnet/iexecutable.h> -#include <vespa/fastos/thread.h> struct DoIt : public FNET_IExecutable { vespalib::Gate gate; @@ -18,10 +17,9 @@ TEST("sync execute") { DoIt exe2; DoIt exe3; DoIt exe4; - FastOS_ThreadPool pool; FNET_Transport transport; ASSERT_TRUE(transport.execute(&exe1)); - ASSERT_TRUE(transport.Start(&pool)); + ASSERT_TRUE(transport.Start()); exe1.gate.await(); ASSERT_TRUE(transport.execute(&exe2)); transport.sync(); @@ -32,7 +30,6 @@ TEST("sync execute") { transport.sync(); transport.WaitFinished(); transport.sync(); - pool.Close(); ASSERT_TRUE(exe1.gate.getCount() == 0u); ASSERT_TRUE(exe2.gate.getCount() == 0u); ASSERT_TRUE(exe3.gate.getCount() == 0u); diff --git a/fnet/src/tests/transport_debugger/transport_debugger_test.cpp b/fnet/src/tests/transport_debugger/transport_debugger_test.cpp index a363b1df4c2..eaf2fd71bde 100644 --- a/fnet/src/tests/transport_debugger/transport_debugger_test.cpp +++ b/fnet/src/tests/transport_debugger/transport_debugger_test.cpp @@ -113,9 +113,9 @@ TEST_FF("transport layers can be run with transport debugger", Fixture(), vespal EXPECT_EQUAL(req4->GetErrorCode(), FRTE_RPC_TIMEOUT); ASSERT_TRUE(req6->CheckReturnTypes("l")); EXPECT_EQUAL(req6->GetReturn()->GetValue(0)._intval64, 8u); - target->SubRef(); - req4->SubRef(); - req6->SubRef(); + target->internal_subref(); + req4->internal_subref(); + req6->internal_subref(); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/fnet/src/vespa/fnet/connection.cpp b/fnet/src/vespa/fnet/connection.cpp index 26367c904b2..fef8a6bf01b 100644 --- a/fnet/src/vespa/fnet/connection.cpp +++ b/fnet/src/vespa/fnet/connection.cpp @@ -63,7 +63,7 @@ struct DoHandshakeWork : vespalib::Executor::Task { DoHandshakeWork(FNET_Connection *conn_in, vespalib::CryptoSocket *socket_in) : conn(conn_in), socket(socket_in) { - conn->AddRef(); + conn->internal_addref(); } void run() override { socket->do_handshake_work(); @@ -82,7 +82,7 @@ FNET_Connection::ResolveHandler::ResolveHandler(FNET_Connection *conn) : connection(conn), address() { - connection->AddRef(); + connection->internal_addref(); } void @@ -94,7 +94,7 @@ FNET_Connection::ResolveHandler::handle_result(vespalib::SocketAddress result) FNET_Connection::ResolveHandler::~ResolveHandler() { - connection->SubRef(); + connection->internal_subref(); } @@ -149,10 +149,9 @@ FNET_Connection::SetState(State state) } if ( ! toDelete.empty() ) { - for (const FNET_Channel::UP & ch : toDelete) { - (void) ch; - SubRef_NoLock(); - } + const uint32_t cnt = toDelete.size(); + const uint32_t reserve = 1; + internal_subref(cnt, reserve); } } @@ -185,14 +184,14 @@ FNET_Connection::HandlePacket(uint32_t plen, uint32_t pcode, _channels.Unregister(channel); if (hp_rc == FNET_IPacketHandler::FNET_FREE_CHANNEL) { - SubRef_NoLock(); + internal_subref(1, 1); toDelete.reset(channel); } } } else if (CanAcceptChannels() && IsFromPeer(chid)) { // open new channel FNET_Channel::UP newChannel(new FNET_Channel(chid, this)); channel = newChannel.get(); - AddRef_NoLock(); + internal_addref(); BeforeCallback(guard, channel); if (_serverAdapter->InitChannel(channel, pcode)) { @@ -203,7 +202,7 @@ FNET_Connection::HandlePacket(uint32_t plen, uint32_t pcode, AfterCallback(guard); if (hp_rc == FNET_IPacketHandler::FNET_FREE_CHANNEL) { - SubRef_NoLock(); + internal_subref(1, 1); } else if (hp_rc == FNET_IPacketHandler::FNET_KEEP_CHANNEL) { _channels.Register(newChannel.release()); } else { @@ -212,7 +211,7 @@ FNET_Connection::HandlePacket(uint32_t plen, uint32_t pcode, } else { AfterCallback(guard); - SubRef_NoLock(); + internal_subref(1, 1); guard.unlock(); LOG(debug, "Connection(%s): channel init failed", GetSpec()); @@ -475,7 +474,7 @@ FNET_Connection::FNET_Connection(FNET_TransportThread *owner, _streamer(streamer), _serverAdapter(serverAdapter), _socket(owner->owner().create_server_crypto_socket(std::move(socket))), - _resolve_handler(nullptr), + _resolve_handler(), _context(), _state(FNET_CONNECTING), _flags(owner->owner().getConfig()), @@ -489,8 +488,7 @@ FNET_Connection::FNET_Connection(FNET_TransportThread *owner, _myQueue(256), _output(0), _channels(), - _callbackTarget(nullptr), - _cleanup(nullptr) + _callbackTarget(nullptr) { assert(_socket && (_socket->get_fd() >= 0)); _num_connections.fetch_add(1, std::memory_order_relaxed); @@ -506,7 +504,7 @@ FNET_Connection::FNET_Connection(FNET_TransportThread *owner, _streamer(streamer), _serverAdapter(serverAdapter), _socket(), - _resolve_handler(nullptr), + _resolve_handler(), _context(context), _state(FNET_CONNECTING), _flags(owner->owner().getConfig()), @@ -520,8 +518,7 @@ FNET_Connection::FNET_Connection(FNET_TransportThread *owner, _myQueue(256), _output(0), _channels(), - _callbackTarget(nullptr), - _cleanup(nullptr) + _callbackTarget(nullptr) { _num_connections.fetch_add(1, std::memory_order_relaxed); } @@ -529,7 +526,7 @@ FNET_Connection::FNET_Connection(FNET_TransportThread *owner, FNET_Connection::~FNET_Connection() { - assert(_cleanup == nullptr); + assert(!_resolve_handler); _num_connections.fetch_sub(1, std::memory_order_relaxed); } @@ -575,12 +572,6 @@ FNET_Connection::handle_handshake_act() return ((GetState() == FNET_CONNECTING) && handshake()); } -void -FNET_Connection::SetCleanupHandler(FNET_IConnectionCleanupHandler *handler) -{ - _cleanup = handler; -} - FNET_Channel* FNET_Connection::OpenChannel(FNET_IPacketHandler *handler, @@ -597,7 +588,7 @@ FNET_Connection::OpenChannel(FNET_IPacketHandler *handler, *chid = newChannel->GetID(); } WaitCallback(guard, nullptr); - AddRef_NoLock(); + internal_addref(); ret = newChannel.release(); _channels.Register(ret); } @@ -613,7 +604,7 @@ FNET_Connection::OpenChannel() { std::lock_guard<std::mutex> guard(_ioc_lock); chid = GetNextID(); - AddRef_NoLock(); + internal_addref(); } return new FNET_Channel(chid, this); } @@ -632,18 +623,20 @@ void FNET_Connection::FreeChannel(FNET_Channel *channel) { delete channel; - SubRef_HasLock(std::unique_lock<std::mutex>(_ioc_lock)); + internal_subref(); } void FNET_Connection::CloseAndFreeChannel(FNET_Channel *channel) { - std::unique_lock<std::mutex> guard(_ioc_lock); - WaitCallback(guard, channel); - _channels.Unregister(channel); - SubRef_HasLock(std::move(guard)); - delete channel; + { + std::unique_lock<std::mutex> guard(_ioc_lock); + WaitCallback(guard, channel); + _channels.Unregister(channel); + delete channel; + } + internal_subref(); } @@ -667,7 +660,7 @@ FNET_Connection::PostPacket(FNET_Packet *packet, uint32_t chid) _writeWork++; _queue.QueuePacket_NoLock(packet, FNET_Context(chid)); if ((writeWork == 0) && (GetState() == FNET_CONNECTED)) { - AddRef_NoLock(); + internal_addref(); guard.unlock(); Owner()->EnableWrite(this, /* needRef = */ false); } @@ -685,16 +678,6 @@ FNET_Connection::Sync() void -FNET_Connection::CleanupHook() -{ - if (_cleanup != nullptr) { - _cleanup->Cleanup(this); - _cleanup = nullptr; - } -} - - -void FNET_Connection::Close() { _resolve_handler.reset(); diff --git a/fnet/src/vespa/fnet/connection.h b/fnet/src/vespa/fnet/connection.h index 10cf74e79de..80927fd375c 100644 --- a/fnet/src/vespa/fnet/connection.h +++ b/fnet/src/vespa/fnet/connection.h @@ -21,30 +21,6 @@ class FNET_IPacketHandler; namespace vespalib::net { class ConnectionAuthContext; } /** - * Interface implemented by objects that want to perform connection - * cleanup. Use the SetCleanupHandler method to register with a - * connection. Currently, there can only be one cleanup handler per - * connection. - **/ -class FNET_IConnectionCleanupHandler -{ -public: - - /** - * Destructor. No cleanup needed for base class. - */ - virtual ~FNET_IConnectionCleanupHandler(void) {} - - /** - * Perform connection cleanup. - * - * @param conn the connection - **/ - virtual void Cleanup(FNET_Connection *conn) = 0; -}; - - -/** * This class represents a single connection with another * computer. The binary format on a connection is defined by the * PacketStreamer given to the constructor. Each connection object may @@ -115,8 +91,6 @@ private: FNET_ChannelLookup _channels; // channel 'DB' FNET_Channel *_callbackTarget; // target of current callback - FNET_IConnectionCleanupHandler *_cleanup; // cleanup handler - std::unique_ptr<vespalib::net::ConnectionAuthContext> _auth_context; static std::atomic<uint64_t> _num_connections; // total number of connections @@ -377,14 +351,6 @@ public: **/ bool handle_handshake_act() override; - /** - * Register a cleanup handler to be invoked when this connection is - * about to be destructed. - * - * @param handler the cleanup handler - **/ - void SetCleanupHandler(FNET_IConnectionCleanupHandler *handler); - /** * Open a new channel on this connection. This method will return @@ -468,14 +434,6 @@ public: /** - * Invoked by the io component superclass before the object is - * destructed. Will invoke the Cleanup method on the cleanup handler - * for this connection, if present. - **/ - void CleanupHook() override; - - - /** * Close this connection immidiately. NOTE: this method should only * be called by the transport thread. **/ diff --git a/fnet/src/vespa/fnet/frt/invoker.cpp b/fnet/src/vespa/fnet/frt/invoker.cpp index d4b35720a8d..421bd0c4d0b 100644 --- a/fnet/src/vespa/fnet/frt/invoker.cpp +++ b/fnet/src/vespa/fnet/frt/invoker.cpp @@ -98,7 +98,7 @@ FRT_RPCInvoker::HandleDone(bool freeChannel) } // send response to client or get rid of it if (_noReply || (_req->GetErrorCode() == FRTE_RPC_BAD_REQUEST)) - _req->SubRef(); + _req->internal_subref(); else ch->Send(_req->CreateReplyPacket()); @@ -128,7 +128,7 @@ void FRT_HookInvoker::Invoke() _req->SetDetachedPT(&detached); (_hook->GetHandler()->*_hook->GetMethod())(_req); assert(!detached); - _req->SubRef(); + _req->internal_subref(); } void diff --git a/fnet/src/vespa/fnet/frt/packets.cpp b/fnet/src/vespa/fnet/frt/packets.cpp index 134a869eafb..5e600c8caa3 100644 --- a/fnet/src/vespa/fnet/frt/packets.cpp +++ b/fnet/src/vespa/fnet/frt/packets.cpp @@ -14,7 +14,7 @@ FRT_RPCPacket::Free() { if (_ownsRef) { _req->DiscardBlobs(); - _req->SubRef(); + _req->internal_subref(); } } diff --git a/fnet/src/vespa/fnet/frt/reflection.cpp b/fnet/src/vespa/fnet/frt/reflection.cpp index af7fa069eb9..c09057ea675 100644 --- a/fnet/src/vespa/fnet/frt/reflection.cpp +++ b/fnet/src/vespa/fnet/frt/reflection.cpp @@ -153,7 +153,7 @@ FRT_ReflectionBuilder::FRT_ReflectionBuilder(FRT_Supervisor *supervisor) FRT_ReflectionBuilder::~FRT_ReflectionBuilder() { Flush(); - _req->SubRef(); + _req->internal_subref(); } diff --git a/fnet/src/vespa/fnet/frt/rpcrequest.cpp b/fnet/src/vespa/fnet/frt/rpcrequest.cpp index ac6dbb26ad6..6870a275ab1 100644 --- a/fnet/src/vespa/fnet/frt/rpcrequest.cpp +++ b/fnet/src/vespa/fnet/frt/rpcrequest.cpp @@ -10,7 +10,6 @@ FRT_RPCRequest::FRT_RPCRequest() _context(), _params(_stash), _return(_stash), - _refcnt(1), _completed(0), _errorCode(FRTE_NO_ERROR), _errorMessageLen(0), @@ -19,14 +18,10 @@ FRT_RPCRequest::FRT_RPCRequest() _methodName(nullptr), _detachedPT(nullptr), _abortHandler(nullptr), - _returnHandler(nullptr), - _cleanupHandler(nullptr) + _returnHandler(nullptr) { } -FRT_RPCRequest::~FRT_RPCRequest() -{ - assert(_refcnt == 0); -} +FRT_RPCRequest::~FRT_RPCRequest() = default; void FRT_RPCRequest::SetError(uint32_t errorCode, const char *errorMessage, uint32_t errorMessageLen) @@ -87,18 +82,9 @@ FRT_RPCRequest::GetConnection() { return _returnHandler->GetConnection(); } -void -FRT_RPCRequest::Cleanup() { - if (_cleanupHandler != nullptr) { - _cleanupHandler->HandleCleanup(); - _cleanupHandler = nullptr; - } -} void FRT_RPCRequest::Reset() { - assert(_refcnt <= 1); - Cleanup(); _context = FNET_Context(); _params.Reset(); _return.Reset(); @@ -118,7 +104,7 @@ FRT_RPCRequest::Reset() { bool FRT_RPCRequest::Recycle() { - if (_refcnt > 1 || _errorCode != FRTE_NO_ERROR) + if (count_refs() > 1 || _errorCode != FRTE_NO_ERROR) return false; Reset(); return true; @@ -126,18 +112,6 @@ FRT_RPCRequest::Recycle() void -FRT_RPCRequest::SubRef() -{ - int oldVal = _refcnt.fetch_sub(1); - assert(oldVal > 0); - if (oldVal == 1) { - Reset(); - delete this; - } -} - - -void FRT_RPCRequest::Print(uint32_t indent) { printf("%*sFRT_RPCRequest {\n", indent, ""); @@ -163,7 +137,7 @@ FRT_RPCRequest::CreateRequestPacket(bool wantReply) flags |= FLAG_FRT_RPC_LITTLE_ENDIAN; if (wantReply) - AddRef(); + internal_addref(); else flags |= FLAG_FRT_RPC_NOREPLY; diff --git a/fnet/src/vespa/fnet/frt/rpcrequest.h b/fnet/src/vespa/fnet/frt/rpcrequest.h index a095c274687..72dae4a6af1 100644 --- a/fnet/src/vespa/fnet/frt/rpcrequest.h +++ b/fnet/src/vespa/fnet/frt/rpcrequest.h @@ -6,6 +6,7 @@ #include "error.h" #include <vespa/fnet/context.h> #include <vespa/vespalib/util/stash.h> +#include <vespa/vespalib/util/ref_counted.h> #include <atomic> class FNET_Packet; @@ -37,20 +38,7 @@ public: }; -class FRT_ICleanupHandler -{ -public: - - /** - * Destructor. No cleanup needed for base class. - */ - virtual ~FRT_ICleanupHandler(void) {} - - virtual void HandleCleanup() = 0; -}; - - -class FRT_RPCRequest +class FRT_RPCRequest : public vespalib::enable_ref_counted { private: using Stash = vespalib::Stash; @@ -58,7 +46,6 @@ private: FNET_Context _context; FRT_Values _params; FRT_Values _return; - std::atomic<int> _refcnt; std::atomic<int> _completed; uint32_t _errorCode; uint32_t _errorMessageLen; @@ -69,7 +56,6 @@ private: bool *_detachedPT; FRT_IAbortHandler *_abortHandler; FRT_IReturnHandler *_returnHandler; - FRT_ICleanupHandler *_cleanupHandler; public: FRT_RPCRequest(const FRT_RPCRequest &) = delete; @@ -86,9 +72,6 @@ public: _return.DiscardBlobs(); } - void AddRef() { _refcnt.fetch_add(1); } - void SubRef(); - void SetContext(FNET_Context context) { _context = context; } FNET_Context GetContext() { return _context; } @@ -137,10 +120,8 @@ public: void SetAbortHandler(FRT_IAbortHandler *handler) { _abortHandler = handler; } void SetReturnHandler(FRT_IReturnHandler *handler) { _returnHandler = handler; } - void SetCleanupHandler(FRT_ICleanupHandler *handler) { _cleanupHandler = handler; } bool Abort(); void Return(); FNET_Connection *GetConnection(); - void Cleanup(); }; diff --git a/fnet/src/vespa/fnet/frt/supervisor.cpp b/fnet/src/vespa/fnet/frt/supervisor.cpp index b08516c5009..7d6b4d727c7 100644 --- a/fnet/src/vespa/fnet/frt/supervisor.cpp +++ b/fnet/src/vespa/fnet/frt/supervisor.cpp @@ -7,7 +7,6 @@ #include <vespa/fnet/transport.h> #include <vespa/fnet/transport_thread.h> #include <vespa/fnet/connector.h> -#include <vespa/fastos/thread.h> #include <vespa/vespalib/util/require.h> FNET_IPacketStreamer * @@ -32,7 +31,7 @@ FRT_Supervisor::~FRT_Supervisor() { _transport->detach(this); if (_connector != nullptr) { - _connector->SubRef(); + _connector->internal_subref(); } } @@ -100,7 +99,7 @@ FRT_Supervisor::AllocRPCRequest(FRT_RPCRequest *tradein) if (tradein->Recycle()) { return tradein; } - tradein->SubRef(); + tradein->internal_subref(); } return new FRT_RPCRequest(); } @@ -114,7 +113,7 @@ FRT_Supervisor::InvokeVoid(FNET_Connection *conn, FRT_RPCRequest *req) ch->Send(req->CreateRequestPacket(false)); ch->Free(); } else { - req->SubRef(); + req->internal_subref(); } } @@ -291,11 +290,10 @@ FRT_Supervisor::SchedulerPtr::SchedulerPtr(FNET_TransportThread *transport_threa namespace fnet::frt { StandaloneFRT::StandaloneFRT(const TransportConfig &config) - : _threadPool(std::make_unique<FastOS_ThreadPool>()), - _transport(std::make_unique<FNET_Transport>(config)), + : _transport(std::make_unique<FNET_Transport>(config)), _supervisor(std::make_unique<FRT_Supervisor>(_transport.get())) { - REQUIRE(_transport->Start(_threadPool.get())); + REQUIRE(_transport->Start()); } StandaloneFRT::StandaloneFRT() diff --git a/fnet/src/vespa/fnet/frt/supervisor.h b/fnet/src/vespa/fnet/frt/supervisor.h index 93272e93b4a..0261c7863b9 100644 --- a/fnet/src/vespa/fnet/frt/supervisor.h +++ b/fnet/src/vespa/fnet/frt/supervisor.h @@ -13,7 +13,6 @@ namespace fnet { class TransportConfig; } class FNET_Transport; class FRT_Target; -class FastOS_ThreadPool; class FNET_Scheduler; class FRT_RPCInvoker; class FRT_IRequestWait; @@ -106,7 +105,6 @@ public: const FRT_Supervisor &supervisor() const { return *_supervisor; } void shutdown(); private: - std::unique_ptr<FastOS_ThreadPool> _threadPool; std::unique_ptr<FNET_Transport> _transport; std::unique_ptr<FRT_Supervisor> _supervisor; }; diff --git a/fnet/src/vespa/fnet/frt/target.cpp b/fnet/src/vespa/fnet/frt/target.cpp index 1645fba91ee..47baa95b9e2 100644 --- a/fnet/src/vespa/fnet/frt/target.cpp +++ b/fnet/src/vespa/fnet/frt/target.cpp @@ -6,7 +6,6 @@ FRT_Target::~FRT_Target() { - assert(_refcnt == 0); FNET_Connection * conn(_conn); _conn = nullptr; if (conn != nullptr) { diff --git a/fnet/src/vespa/fnet/frt/target.h b/fnet/src/vespa/fnet/frt/target.h index ac6bf377a14..773daaca8b1 100644 --- a/fnet/src/vespa/fnet/frt/target.h +++ b/fnet/src/vespa/fnet/frt/target.h @@ -3,16 +3,16 @@ #pragma once #include <vespa/fnet/connection.h> +#include <vespa/vespalib/util/ref_counted.h> #include <atomic> class FNET_Scheduler; class FRT_RPCRequest; class FRT_IRequestWait; -class FRT_Target +class FRT_Target : public vespalib::enable_ref_counted { private: - std::atomic<int> _refcnt; FNET_Scheduler *_scheduler; FNET_Connection *_conn; @@ -21,23 +21,12 @@ private: public: FRT_Target(FNET_Scheduler *scheduler, FNET_Connection *conn) - : _refcnt(1), - _scheduler(scheduler), + : _scheduler(scheduler), _conn(conn) {} ~FRT_Target(); FNET_Connection *GetConnection() const { return _conn; } - - void AddRef() { _refcnt.fetch_add(1); } - void SubRef() { - if (_refcnt.fetch_sub(1) == 1) { - delete this; - } - } - - int GetRefCnt() const { return _refcnt; } - bool IsValid() { return ((_conn != nullptr) && (_conn->GetState() <= FNET_Connection::FNET_CONNECTED)); diff --git a/fnet/src/vespa/fnet/iocomponent.cpp b/fnet/src/vespa/fnet/iocomponent.cpp index f08718c0c5c..c4fc7d859d4 100644 --- a/fnet/src/vespa/fnet/iocomponent.cpp +++ b/fnet/src/vespa/fnet/iocomponent.cpp @@ -16,7 +16,6 @@ FNET_IOComponent::FNET_IOComponent(FNET_TransportThread *owner, _ioc_spec(spec), _flags(shouldTimeOut), _ioc_socket_fd(socket_fd), - _ioc_refcnt(1), _ioc_timestamp(vespalib::steady_clock::now()), _ioc_lock(), _ioc_cond() @@ -39,58 +38,6 @@ FNET_IOComponent::UpdateTimeOut() { _ioc_owner->UpdateTimeOut(this); } -void -FNET_IOComponent::AddRef() -{ - std::lock_guard<std::mutex> guard(_ioc_lock); - assert(_ioc_refcnt > 0); - _ioc_refcnt++; -} - - -void -FNET_IOComponent::AddRef_NoLock() -{ - assert(_ioc_refcnt > 0); - _ioc_refcnt++; -} - - -void -FNET_IOComponent::SubRef() -{ - { - std::lock_guard<std::mutex> guard(_ioc_lock); - assert(_ioc_refcnt > 0); - if (--_ioc_refcnt > 0) { - return; - } - } - CleanupHook(); - delete this; -} - - -void -FNET_IOComponent::SubRef_HasLock(std::unique_lock<std::mutex> guard) -{ - assert(_ioc_refcnt > 0); - if (--_ioc_refcnt > 0) { - return; - } - guard.unlock(); - CleanupHook(); - delete this; -} - - -void -FNET_IOComponent::SubRef_NoLock() -{ - assert(_ioc_refcnt > 1); - _ioc_refcnt--; -} - void FNET_IOComponent::attach_selector(Selector &selector) @@ -141,8 +88,3 @@ FNET_IOComponent::handle_handshake_act() { return true; } - -void -FNET_IOComponent::CleanupHook() -{ -} diff --git a/fnet/src/vespa/fnet/iocomponent.h b/fnet/src/vespa/fnet/iocomponent.h index 106e31c9236..b88b2700db5 100644 --- a/fnet/src/vespa/fnet/iocomponent.h +++ b/fnet/src/vespa/fnet/iocomponent.h @@ -4,6 +4,7 @@ #include "scheduler.h" #include <vespa/vespalib/net/selector.h> +#include <vespa/vespalib/util/ref_counted.h> #include <mutex> #include <condition_variable> #include <chrono> @@ -18,7 +19,7 @@ class FNET_Config; * Components do IO against the network and that they use sockets to * perform that IO. **/ -class FNET_IOComponent +class FNET_IOComponent : public vespalib::enable_ref_counted { friend class FNET_TransportThread; @@ -46,7 +47,6 @@ protected: std::string _ioc_spec; // connect/listen spec Flags _flags; // Compressed representation of boolean flags; int _ioc_socket_fd; // source of events. - uint32_t _ioc_refcnt; // reference counter vespalib::steady_time _ioc_timestamp; // last I/O activity std::mutex _ioc_lock; // synchronization std::condition_variable _ioc_cond; // synchronization @@ -88,43 +88,6 @@ public: std::unique_lock<std::mutex> getGuard() { return std::unique_lock<std::mutex>(_ioc_lock); } /** - * Allocate a reference to this component. This method locks the - * object to protect the reference counter. - **/ - void AddRef(); - - - /** - * Allocate a reference to this component without locking the - * object. Caller already has lock on object. - **/ - void AddRef_NoLock(); - - - /** - * Free a reference to this component. This method locks the object - * to protect the reference counter. - **/ - void SubRef(); - - - /** - * Free a reference to this component. This method uses locking to - * protect the reference counter, but assumes that the lock has - * already been obtained when this method is called. - **/ - void SubRef_HasLock(std::unique_lock<std::mutex> guard); - - - /** - * Free a reference to this component without locking the - * object. NOTE: this method may only be called on objects with more - * than one reference. - **/ - void SubRef_NoLock(); - - - /** * @return the owning TransportThread object. **/ FNET_TransportThread *Owner() { return _ioc_owner; } @@ -217,13 +180,6 @@ public: **/ virtual bool handle_handshake_act(); - /** - * This method is called by the SubRef methods just before the - * object is deleted. It may be used to perform cleanup tasks that - * must be done before the destructor is invoked. - **/ - virtual void CleanupHook(); - /** * Close this component immediately. NOTE: this method should only diff --git a/fnet/src/vespa/fnet/transport.cpp b/fnet/src/vespa/fnet/transport.cpp index ae864ea0821..1553fc010c0 100644 --- a/fnet/src/vespa/fnet/transport.cpp +++ b/fnet/src/vespa/fnet/transport.cpp @@ -136,6 +136,7 @@ FNET_Transport::FNET_Transport(const fnet::TransportConfig &cfg) _time_tools(cfg.time_tools()), _work_pool(std::make_unique<vespalib::ThreadStackExecutor>(1, fnet_work_pool, 1024)), _threads(), + _pool(), _config(cfg.config()) { // TODO Temporary logging to track down overspend @@ -146,7 +147,10 @@ FNET_Transport::FNET_Transport(const fnet::TransportConfig &cfg) } } -FNET_Transport::~FNET_Transport() = default; +FNET_Transport::~FNET_Transport() +{ + _pool.join(); +} void FNET_Transport::post_or_perform(vespalib::Executor::Task::UP task) @@ -266,13 +270,12 @@ FNET_Transport::WaitFinished() } bool -FNET_Transport::Start(FastOS_ThreadPool *pool) +FNET_Transport::Start() { - bool result = true; for (const auto &thread: _threads) { - result &= thread->Start(pool); + thread->Start(_pool); } - return result; + return true; } void diff --git a/fnet/src/vespa/fnet/transport.h b/fnet/src/vespa/fnet/transport.h index 3f9328296d6..d658059f0bb 100644 --- a/fnet/src/vespa/fnet/transport.h +++ b/fnet/src/vespa/fnet/transport.h @@ -7,9 +7,9 @@ #include <vespa/vespalib/net/async_resolver.h> #include <vespa/vespalib/net/crypto_engine.h> #include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/util/thread.h> class FNET_TransportThread; -class FastOS_ThreadPool; class FNET_Connector; class FNET_IPacketStreamer; class FNET_IServerAdapter; @@ -111,6 +111,7 @@ private: fnet::TimeTools::SP _time_tools; std::unique_ptr<vespalib::SyncableThreadExecutor> _work_pool; Threads _threads; + vespalib::ThreadPool _pool; const FNET_Config _config; /** @@ -317,9 +318,8 @@ public: * ok. * * @return thread create status. - * @param pool threadpool that may be used to spawn new threads. **/ - bool Start(FastOS_ThreadPool *pool); + bool Start(); /** * Capture transport threads. Used for testing purposes, diff --git a/fnet/src/vespa/fnet/transport_thread.cpp b/fnet/src/vespa/fnet/transport_thread.cpp index 262cdaec190..0b0df02c04c 100644 --- a/fnet/src/vespa/fnet/transport_thread.cpp +++ b/fnet/src/vespa/fnet/transport_thread.cpp @@ -107,7 +107,7 @@ FNET_TransportThread::FlushDeleteList() FNET_IOComponent *tmp = _deleteList; _deleteList = tmp->_ioc_next; assert(tmp->_flags._ioc_delete); - tmp->SubRef(); + tmp->internal_subref(); } } @@ -143,12 +143,12 @@ FNET_TransportThread::DiscardEvent(FNET_ControlPacket *cpacket, switch (cpacket->GetCommand()) { case FNET_ControlPacket::FNET_CMD_IOC_ADD: context._value.IOC->Close(); - context._value.IOC->SubRef(); + context._value.IOC->internal_subref(); break; case FNET_ControlPacket::FNET_CMD_IOC_ENABLE_WRITE: case FNET_ControlPacket::FNET_CMD_IOC_HANDSHAKE_ACT: case FNET_ControlPacket::FNET_CMD_IOC_CLOSE: - context._value.IOC->SubRef(); + context._value.IOC->internal_subref(); break; } } @@ -173,7 +173,7 @@ FNET_TransportThread::handle_close_cmd(FNET_IOComponent *ioc) { if (ioc->_flags._ioc_added) { RemoveComponent(ioc); - ioc->SubRef(); + ioc->internal_subref(); } ioc->Close(); AddDeleteComponent(ioc); @@ -288,7 +288,7 @@ FNET_TransportThread::Listen(const char *spec, FNET_IPacketStreamer *streamer, if (server_socket.valid() && server_socket.set_blocking(false)) { FNET_Connector *connector = new FNET_Connector(this, streamer, serverAdapter, spec, std::move(server_socket)); connector->EnableReadEvent(true); - connector->AddRef_NoLock(); + connector->internal_addref(); Add(connector, /* needRef = */ false); return connector; } @@ -314,7 +314,7 @@ void FNET_TransportThread::Add(FNET_IOComponent *comp, bool needRef) { if (needRef) { - comp->AddRef(); + comp->internal_addref(); } PostEvent(&FNET_ControlPacket::IOCAdd, FNET_Context(comp)); } @@ -324,7 +324,7 @@ void FNET_TransportThread::EnableWrite(FNET_IOComponent *comp, bool needRef) { if (needRef) { - comp->AddRef(); + comp->internal_addref(); } PostEvent(&FNET_ControlPacket::IOCEnableWrite, FNET_Context(comp)); } @@ -333,7 +333,7 @@ void FNET_TransportThread::handshake_act(FNET_IOComponent *comp, bool needRef) { if (needRef) { - comp->AddRef(); + comp->internal_addref(); } PostEvent(&FNET_ControlPacket::IOCHandshakeACT, FNET_Context(comp)); } @@ -342,7 +342,7 @@ void FNET_TransportThread::Close(FNET_IOComponent *comp, bool needRef) { if (needRef) { - comp->AddRef(); + comp->internal_addref(); } PostEvent(&FNET_ControlPacket::IOCClose, FNET_Context(comp)); } @@ -449,7 +449,7 @@ FNET_TransportThread::handle_wakeup() } if (context._value.IOC->_flags._ioc_delete) { - context._value.IOC->SubRef(); + context._value.IOC->internal_subref(); continue; } @@ -460,14 +460,14 @@ FNET_TransportThread::handle_wakeup() case FNET_ControlPacket::FNET_CMD_IOC_ENABLE_WRITE: context._value.IOC->EnableWriteEvent(true); if (context._value.IOC->HandleWriteEvent()) { - context._value.IOC->SubRef(); + context._value.IOC->internal_subref(); } else { handle_close_cmd(context._value.IOC); } break; case FNET_ControlPacket::FNET_CMD_IOC_HANDSHAKE_ACT: if (context._value.IOC->handle_handshake_act()) { - context._value.IOC->SubRef(); + context._value.IOC->internal_subref(); } else { handle_close_cmd(context._value.IOC); } @@ -577,7 +577,7 @@ FNET_TransportThread::endEventLoop() { component = component->_ioc_next; RemoveComponent(tmp); tmp->Close(); - tmp->SubRef(); + tmp->internal_subref(); } assert(_componentsHead == nullptr && _componentsTail == nullptr && @@ -598,28 +598,28 @@ FNET_TransportThread::endEventLoop() { bool -FNET_TransportThread::Start(FastOS_ThreadPool *pool) +FNET_TransportThread::Start(vespalib::ThreadPool &pool) { - return (pool != nullptr && pool->NewThread(this)); + pool.start([this](){run();}); + return true; } void FNET_TransportThread::Main() { - Run(nullptr, nullptr); + run(); } void -FNET_TransportThread::Run(FastOS_ThreadInterface *thisThread, void *) +FNET_TransportThread::run() { if (!InitEventLoop()) { LOG(warning, "Transport: Run: Could not init event loop"); return; } while (EventLoopIteration()) { - if (thisThread != nullptr && thisThread->GetBreakFlag()) - ShutDown(false); + // event loop must be stopped from the outside } } diff --git a/fnet/src/vespa/fnet/transport_thread.h b/fnet/src/vespa/fnet/transport_thread.h index 1911b11a81c..6047d4e3482 100644 --- a/fnet/src/vespa/fnet/transport_thread.h +++ b/fnet/src/vespa/fnet/transport_thread.h @@ -6,9 +6,9 @@ #include "config.h" #include "task.h" #include "packetqueue.h" -#include <vespa/fastos/thread.h> #include <vespa/vespalib/net/socket_handle.h> #include <vespa/vespalib/net/selector.h> +#include <vespa/vespalib/util/thread.h> #include <atomic> #include <mutex> #include <condition_variable> @@ -26,7 +26,7 @@ class FNET_IServerAdapter; * the network related work for the application in both client and * server aspects. **/ -class FNET_TransportThread : public FastOS_Runnable +class FNET_TransportThread { friend class FNET_IOComponent; @@ -97,7 +97,7 @@ private: /** - * Delete (call SubRef on) all IO Components in the delete list. + * Delete (call internal_subref on) all IO Components in the delete list. **/ void FlushDeleteList(); @@ -195,7 +195,7 @@ public: * Destruct object. This should NOT be done before the transport * thread has completed it's work and raised the finished flag. **/ - ~FNET_TransportThread() override; + ~FNET_TransportThread(); /** @@ -277,7 +277,7 @@ public: * @param needRef should be set to false if the caller of this * method already has obtained an extra reference to the * component. If this flag is true, this method will call the - * AddRef method on the component. + * internal_addref method on the component. **/ void Add(FNET_IOComponent *comp, bool needRef = true); @@ -294,7 +294,7 @@ public: * @param needRef should be set to false if the caller of this * method already has obtained an extra reference to the * component. If this flag is true, this method will call the - * AddRef method on the component. + * internal_addref method on the component. **/ void EnableWrite(FNET_IOComponent *comp, bool needRef = true); @@ -312,7 +312,7 @@ public: * @param needRef should be set to false if the caller of this * method already has obtained an extra reference to the * component. If this flag is true, this method will call the - * AddRef method on the component. + * internal_addref method on the component. **/ void handshake_act(FNET_IOComponent *comp, bool needRef = true); @@ -330,7 +330,7 @@ public: * @param needRef should be set to false if the caller of this * method already has obtained an extra reference to the * component. If this flag is true, this method will call the - * AddRef method on the component. + * internal_addref method on the component. **/ void Close(FNET_IOComponent *comp, bool needRef = true); @@ -425,7 +425,7 @@ public: * @return thread create status. * @param pool threadpool that may be used to spawn a new thread. **/ - bool Start(FastOS_ThreadPool *pool); + bool Start(vespalib::ThreadPool &pool); /** @@ -440,5 +440,5 @@ public: * This is where the transport thread lives, when started by * invoking one of the @ref Main or @ref Start methods. **/ - void Run(FastOS_ThreadInterface *thisThread, void *args) override; + void run(); }; diff --git a/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/VespaAwsCredentialsProvider.java b/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/VespaAwsCredentialsProvider.java index 1991ff3ad88..a812ef4efe3 100644 --- a/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/VespaAwsCredentialsProvider.java +++ b/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/VespaAwsCredentialsProvider.java @@ -39,7 +39,7 @@ public class VespaAwsCredentialsProvider implements AWSCredentialsProvider { try { credentials.set(readCredentials()); } catch (Exception e) { - throw new RuntimeException("Unable to get credentials in " + credentialsPath.toString(), e); + throw new RuntimeException("Unable to get credentials. Please ensure cluster is configured as exclusive. See: https://cloud.vespa.ai/en/reference/services#nodes"); } } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/MetricProvider.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/MetricProvider.java index 16d4861b244..c206393400d 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/MetricProvider.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/MetricProvider.java @@ -21,4 +21,5 @@ public class MetricProvider implements Provider<Metric> { public Metric get() { return new MetricImpl(consumerProvider); } + } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java index 53acd8cbb1b..1ae25aa0cfa 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java @@ -45,6 +45,7 @@ public class ExportPackages { } } + // Make sure to update the junit integration test `ExportPackagesIT.java` if the set of exported packages is modified. private static String getExportPackages(String[] jars) throws IOException { StringBuilder out = new StringBuilder(); out.append(getSystemPackages()).append(", ") @@ -53,6 +54,7 @@ public class ExportPackages { .append("com.yahoo.jdisc.handler, ") .append("com.yahoo.jdisc.service, ") .append("com.yahoo.jdisc.statistics, ") + .append("com.yahoo.jdisc.refcount, ") .append("javax.inject;version=1.0.0, ") // TODO Vespa 9: remove. Included in guice, but not exported. Needed by container-jersey. .append("org.aopalliance.intercept, ") diff --git a/jdisc_core/src/test/resources/exportPackages.properties b/jdisc_core/src/test/resources/exportPackages.properties index 82242b1644b..b49f91842e3 100644 --- a/jdisc_core/src/test/resources/exportPackages.properties +++ b/jdisc_core/src/test/resources/exportPackages.properties @@ -1,3 +1,3 @@ #generated by com.yahoo.jdisc.core.ExportPackages #Wed Jul 20 02:55:26 CEST 2022 -exportPackages=org.osgi.framework; version\="1.10.0", org.osgi.framework.connect; version\="1.0.0", org.osgi.framework.dto; uses\:\="org.osgi.dto"; version\="1.8.0", org.osgi.framework.hooks.bundle; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.resolver; uses\:\="org.osgi.framework.wiring"; version\="1.0.0", org.osgi.framework.hooks.service; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.weaving; uses\:\="org.osgi.framework.wiring"; version\="1.1.0", org.osgi.framework.launch; uses\:\="org.osgi.framework"; version\="1.2.0", org.osgi.framework.namespace; uses\:\="org.osgi.resource"; version\="1.2.0", org.osgi.framework.startlevel; uses\:\="org.osgi.framework"; version\="1.0.0", org.osgi.framework.startlevel.dto; uses\:\="org.osgi.dto"; version\="1.0.0", org.osgi.framework.wiring; uses\:\="org.osgi.framework,org.osgi.resource"; version\="1.2.0", org.osgi.framework.wiring.dto; uses\:\="org.osgi.dto,org.osgi.resource.dto"; version\="1.3.0", org.osgi.resource; version\="1.0.1", org.osgi.resource.dto; uses\:\="org.osgi.dto"; version\="1.0.1", org.osgi.service.packageadmin; uses\:\="org.osgi.framework"; version\="1.2.1", org.osgi.service.startlevel; uses\:\="org.osgi.framework"; version\="1.1.1", org.osgi.service.url; version\="1.0.1", org.osgi.service.resolver; uses\:\="org.osgi.resource"; version\="1.1.1", org.osgi.util.tracker; uses\:\="org.osgi.framework"; version\="1.5.3", org.osgi.dto; version\="1.1.1", org.osgi.service.condition; version\="1.0.0", javax.security.auth.callback; version\="0.0.0.JavaSE_017", javax.security.auth.login; version\="0.0.0.JavaSE_017", javax.net.ssl; version\="0.0.0.JavaSE_017", java.lang.constant; version\="0.0.0.JavaSE_017", java.security.interfaces; version\="0.0.0.JavaSE_017", java.text.spi; version\="0.0.0.JavaSE_017", java.nio.channels.spi; version\="0.0.0.JavaSE_017", java.math; version\="0.0.0.JavaSE_017", java.nio.file; version\="0.0.0.JavaSE_017", java.util.concurrent.atomic; version\="0.0.0.JavaSE_017", java.security.cert; version\="0.0.0.JavaSE_017", java.security.spec; version\="0.0.0.JavaSE_017", java.nio.channels; version\="0.0.0.JavaSE_017", java.time.chrono; version\="0.0.0.JavaSE_017", javax.crypto; version\="0.0.0.JavaSE_017", java.time.zone; version\="0.0.0.JavaSE_017", java.nio.charset; version\="0.0.0.JavaSE_017", java.io; version\="0.0.0.JavaSE_017", java.util.spi; version\="0.0.0.JavaSE_017", java.net; version\="0.0.0.JavaSE_017", javax.security.cert; version\="0.0.0.JavaSE_017", java.lang.annotation; version\="0.0.0.JavaSE_017", javax.security.auth.spi; version\="0.0.0.JavaSE_017", java.util.concurrent; version\="0.0.0.JavaSE_017", java.nio.charset.spi; version\="0.0.0.JavaSE_017", javax.crypto.interfaces; version\="0.0.0.JavaSE_017", java.util; version\="0.0.0.JavaSE_017", java.security; version\="0.0.0.JavaSE_017", java.nio.file.spi; version\="0.0.0.JavaSE_017", java.nio; version\="0.0.0.JavaSE_017", java.util.jar; version\="0.0.0.JavaSE_017", javax.security.auth; version\="0.0.0.JavaSE_017", java.lang.ref; version\="0.0.0.JavaSE_017", java.util.regex; version\="0.0.0.JavaSE_017", java.net.spi; version\="0.0.0.JavaSE_017", java.lang.module; version\="0.0.0.JavaSE_017", java.lang.invoke; version\="0.0.0.JavaSE_017", java.time.format; version\="0.0.0.JavaSE_017", java.util.concurrent.locks; version\="0.0.0.JavaSE_017", java.time.temporal; version\="0.0.0.JavaSE_017", java.util.zip; version\="0.0.0.JavaSE_017", java.nio.file.attribute; version\="0.0.0.JavaSE_017", java.util.random; version\="0.0.0.JavaSE_017", java.text; version\="0.0.0.JavaSE_017", javax.crypto.spec; version\="0.0.0.JavaSE_017", java.util.stream; version\="0.0.0.JavaSE_017", java.time; version\="0.0.0.JavaSE_017", java.lang; version\="0.0.0.JavaSE_017", java.lang.runtime; version\="0.0.0.JavaSE_017", java.util.function; version\="0.0.0.JavaSE_017", javax.net; version\="0.0.0.JavaSE_017", javax.security.auth.x500; version\="0.0.0.JavaSE_017", java.lang.reflect; version\="0.0.0.JavaSE_017", javax.lang.model; version\="0.0.0.JavaSE_017", javax.annotation.processing; version\="0.0.0.JavaSE_017", javax.lang.model.element; version\="0.0.0.JavaSE_017", javax.tools; version\="0.0.0.JavaSE_017", javax.lang.model.type; version\="0.0.0.JavaSE_017", javax.lang.model.util; version\="0.0.0.JavaSE_017", java.awt.datatransfer; version\="0.0.0.JavaSE_017", java.awt.desktop; version\="0.0.0.JavaSE_017", javax.swing.plaf.synth; version\="0.0.0.JavaSE_017", java.beans; version\="0.0.0.JavaSE_017", javax.swing.text.html.parser; version\="0.0.0.JavaSE_017", javax.swing.text.rtf; version\="0.0.0.JavaSE_017", java.awt.font; version\="0.0.0.JavaSE_017", javax.imageio; version\="0.0.0.JavaSE_017", java.awt.im.spi; version\="0.0.0.JavaSE_017", java.applet; version\="0.0.0.JavaSE_017", javax.sound.midi; version\="0.0.0.JavaSE_017", java.awt.dnd; version\="0.0.0.JavaSE_017", javax.swing.text; version\="0.0.0.JavaSE_017", javax.swing.plaf.basic; version\="0.0.0.JavaSE_017", javax.swing.undo; version\="0.0.0.JavaSE_017", javax.swing.plaf; version\="0.0.0.JavaSE_017", javax.swing.filechooser; version\="0.0.0.JavaSE_017", javax.imageio.event; version\="0.0.0.JavaSE_017", javax.sound.sampled; version\="0.0.0.JavaSE_017", javax.print.attribute; version\="0.0.0.JavaSE_017", javax.print; version\="0.0.0.JavaSE_017", javax.swing.plaf.nimbus; version\="0.0.0.JavaSE_017", javax.accessibility; version\="0.0.0.JavaSE_017", java.awt.event; version\="0.0.0.JavaSE_017", javax.swing.text.html; version\="0.0.0.JavaSE_017", javax.imageio.spi; version\="0.0.0.JavaSE_017", javax.swing.border; version\="0.0.0.JavaSE_017", javax.sound.sampled.spi; version\="0.0.0.JavaSE_017", javax.imageio.plugins.bmp; version\="0.0.0.JavaSE_017", java.awt.color; version\="0.0.0.JavaSE_017", java.awt.geom; version\="0.0.0.JavaSE_017", javax.imageio.plugins.jpeg; version\="0.0.0.JavaSE_017", javax.swing.tree; version\="0.0.0.JavaSE_017", javax.imageio.plugins.tiff; version\="0.0.0.JavaSE_017", java.awt.print; version\="0.0.0.JavaSE_017", java.awt.image; version\="0.0.0.JavaSE_017", javax.imageio.metadata; version\="0.0.0.JavaSE_017", javax.swing.table; version\="0.0.0.JavaSE_017", javax.sound.midi.spi; version\="0.0.0.JavaSE_017", javax.print.attribute.standard; version\="0.0.0.JavaSE_017", javax.swing.colorchooser; version\="0.0.0.JavaSE_017", javax.swing; version\="0.0.0.JavaSE_017", java.awt.image.renderable; version\="0.0.0.JavaSE_017", javax.swing.plaf.multi; version\="0.0.0.JavaSE_017", java.awt.im; version\="0.0.0.JavaSE_017", javax.print.event; version\="0.0.0.JavaSE_017", javax.swing.plaf.metal; version\="0.0.0.JavaSE_017", java.beans.beancontext; version\="0.0.0.JavaSE_017", java.awt; version\="0.0.0.JavaSE_017", javax.imageio.stream; version\="0.0.0.JavaSE_017", javax.swing.event; version\="0.0.0.JavaSE_017", java.lang.instrument; version\="0.0.0.JavaSE_017", java.util.logging; version\="0.0.0.JavaSE_017", javax.management.timer; version\="0.0.0.JavaSE_017", javax.management; version\="0.0.0.JavaSE_017", javax.management.relation; version\="0.0.0.JavaSE_017", javax.management.loading; version\="0.0.0.JavaSE_017", javax.management.openmbean; version\="0.0.0.JavaSE_017", java.lang.management; version\="0.0.0.JavaSE_017", javax.management.remote; version\="0.0.0.JavaSE_017", javax.management.monitor; version\="0.0.0.JavaSE_017", javax.management.modelmbean; version\="0.0.0.JavaSE_017", javax.management.remote.rmi; version\="0.0.0.JavaSE_017", javax.naming.event; version\="0.0.0.JavaSE_017", javax.naming.ldap.spi; version\="0.0.0.JavaSE_017", javax.naming; version\="0.0.0.JavaSE_017", javax.naming.spi; version\="0.0.0.JavaSE_017", javax.naming.ldap; version\="0.0.0.JavaSE_017", javax.naming.directory; version\="0.0.0.JavaSE_017", java.net.http; version\="0.0.0.JavaSE_017", java.util.prefs; version\="0.0.0.JavaSE_017", java.rmi.registry; version\="0.0.0.JavaSE_017", javax.rmi.ssl; version\="0.0.0.JavaSE_017", java.rmi.dgc; version\="0.0.0.JavaSE_017", java.rmi; version\="0.0.0.JavaSE_017", java.rmi.server; version\="0.0.0.JavaSE_017", javax.script; version\="0.0.0.JavaSE_017", org.ietf.jgss; version\="0.0.0.JavaSE_017", javax.security.auth.kerberos; version\="0.0.0.JavaSE_017", javax.security.sasl; version\="0.0.0.JavaSE_017", javax.smartcardio; version\="0.0.0.JavaSE_017", java.sql; version\="0.0.0.JavaSE_017", javax.sql; version\="0.0.0.JavaSE_017", javax.sql.rowset.spi; version\="0.0.0.JavaSE_017", javax.sql.rowset.serial; version\="0.0.0.JavaSE_017", javax.sql.rowset; version\="0.0.0.JavaSE_017", javax.transaction.xa; version\="0.0.0.JavaSE_017", javax.xml; version\="0.0.0.JavaSE_017", javax.xml.transform.sax; version\="0.0.0.JavaSE_017", javax.xml.datatype; version\="0.0.0.JavaSE_017", javax.xml.catalog; version\="0.0.0.JavaSE_017", org.w3c.dom.traversal; version\="0.0.0.JavaSE_017", javax.xml.stream.events; version\="0.0.0.JavaSE_017", javax.xml.stream; version\="0.0.0.JavaSE_017", org.xml.sax; version\="0.0.0.JavaSE_017", javax.xml.transform; version\="0.0.0.JavaSE_017", javax.xml.xpath; version\="0.0.0.JavaSE_017", org.w3c.dom.events; version\="0.0.0.JavaSE_017", org.w3c.dom.ranges; version\="0.0.0.JavaSE_017", org.w3c.dom.ls; version\="0.0.0.JavaSE_017", javax.xml.stream.util; version\="0.0.0.JavaSE_017", javax.xml.namespace; version\="0.0.0.JavaSE_017", javax.xml.transform.stax; version\="0.0.0.JavaSE_017", org.xml.sax.helpers; version\="0.0.0.JavaSE_017", org.w3c.dom.views; version\="0.0.0.JavaSE_017", org.w3c.dom.bootstrap; version\="0.0.0.JavaSE_017", org.w3c.dom; version\="0.0.0.JavaSE_017", javax.xml.transform.stream; version\="0.0.0.JavaSE_017", javax.xml.transform.dom; version\="0.0.0.JavaSE_017", javax.xml.validation; version\="0.0.0.JavaSE_017", javax.xml.parsers; version\="0.0.0.JavaSE_017", org.xml.sax.ext; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.spec; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.keyinfo; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig; version\="0.0.0.JavaSE_017", com.sun.java.accessibility.util; version\="0.0.0.JavaSE_017", com.sun.tools.attach.spi; version\="0.0.0.JavaSE_017", com.sun.tools.attach; version\="0.0.0.JavaSE_017", com.sun.source.doctree; version\="0.0.0.JavaSE_017", com.sun.source.tree; version\="0.0.0.JavaSE_017", com.sun.source.util; version\="0.0.0.JavaSE_017", com.sun.tools.javac; version\="0.0.0.JavaSE_017", jdk.dynalink.beans; version\="0.0.0.JavaSE_017", jdk.dynalink.linker.support; version\="0.0.0.JavaSE_017", jdk.dynalink.support; version\="0.0.0.JavaSE_017", jdk.dynalink; version\="0.0.0.JavaSE_017", jdk.dynalink.linker; version\="0.0.0.JavaSE_017", com.sun.net.httpserver; version\="0.0.0.JavaSE_017", com.sun.net.httpserver.spi; version\="0.0.0.JavaSE_017", com.sun.jarsigner; version\="0.0.0.JavaSE_017", jdk.security.jarsigner; version\="0.0.0.JavaSE_017", jdk.javadoc.doclet; version\="0.0.0.JavaSE_017", com.sun.tools.jconsole; version\="0.0.0.JavaSE_017", com.sun.jdi.connect; version\="0.0.0.JavaSE_017", com.sun.jdi.event; version\="0.0.0.JavaSE_017", com.sun.jdi.connect.spi; version\="0.0.0.JavaSE_017", com.sun.jdi; version\="0.0.0.JavaSE_017", com.sun.jdi.request; version\="0.0.0.JavaSE_017", jdk.jfr.consumer; version\="0.0.0.JavaSE_017", jdk.jfr; version\="0.0.0.JavaSE_017", jdk.jshell; version\="0.0.0.JavaSE_017", jdk.jshell.execution; version\="0.0.0.JavaSE_017", jdk.jshell.spi; version\="0.0.0.JavaSE_017", jdk.jshell.tool; version\="0.0.0.JavaSE_017", netscape.javascript; version\="0.0.0.JavaSE_017", com.sun.management; version\="0.0.0.JavaSE_017", jdk.management.jfr; version\="0.0.0.JavaSE_017", jdk.net; version\="0.0.0.JavaSE_017", jdk.nio; version\="0.0.0.JavaSE_017", jdk.nio.mapmode; version\="0.0.0.JavaSE_017", com.sun.nio.sctp; version\="0.0.0.JavaSE_017", com.sun.security.auth.module; version\="0.0.0.JavaSE_017", com.sun.security.auth.login; version\="0.0.0.JavaSE_017", com.sun.security.auth; version\="0.0.0.JavaSE_017", com.sun.security.auth.callback; version\="0.0.0.JavaSE_017", com.sun.security.jgss; version\="0.0.0.JavaSE_017", sun.reflect; version\="0.0.0.JavaSE_017", sun.misc; version\="0.0.0.JavaSE_017", com.sun.nio.file; version\="0.0.0.JavaSE_017", jdk.swing.interop; version\="0.0.0.JavaSE_017", org.w3c.dom.stylesheets; version\="0.0.0.JavaSE_017", org.w3c.dom.html; version\="0.0.0.JavaSE_017", org.w3c.dom.xpath; version\="0.0.0.JavaSE_017", org.w3c.dom.css; version\="0.0.0.JavaSE_017", com.yahoo.jdisc, com.yahoo.jdisc.application, com.yahoo.jdisc.handler, com.yahoo.jdisc.service, com.yahoo.jdisc.statistics, javax.inject;version\=1.0.0, org.aopalliance.intercept, org.aopalliance.aop, com.google.common.annotations;version\="27.1.0",com.google.common.base;version\="27.1.0",com.google.common.cache;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent",com.google.common.collect;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.escape;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.eventbus;version\="27.1.0",com.google.common.graph;version\="27.1.0";uses\:\="com.google.common.collect",com.google.common.hash;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.html;version\="27.1.0";uses\:\="com.google.common.escape",com.google.common.io;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.graph,com.google.common.hash",com.google.common.math;version\="27.1.0",com.google.common.net;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.escape",com.google.common.primitives;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.reflect;version\="27.1.0";uses\:\="com.google.common.collect,com.google.common.io",com.google.common.util.concurrent;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent.internal",com.google.common.xml;version\="27.1.0";uses\:\="com.google.common.escape", com.google.inject;version\="1.4",com.google.inject.binder;version\="1.4",com.google.inject.matcher;version\="1.4",com.google.inject.multibindings;version\="1.4",com.google.inject.name;version\="1.4",com.google.inject.spi;version\="1.4",com.google.inject.util;version\="1.4", org.slf4j;version\=1.7.32, org.slf4j.spi;version\=1.7.32, org.slf4j.helpers;version\=1.7.32, org.slf4j.event;version\=1.7.32, org.slf4j.impl;version\=1.7.32, org.apache.commons.logging;version\=1.2, org.apache.commons.logging.impl;version\=1.2, org.apache.log4j;version\=1.2.17,org.apache.log4j.helpers;version\=1.2.17,org.apache.log4j.spi;version\=1.2.17,org.apache.log4j.xml;version\=1.2.17, com.yahoo.component.annotation;version\="1.0.0", com.yahoo.config;version\=1.0.0, com.yahoo.vespa.defaults;version\=1.0.0, ai.vespa.http;version\=1.0.0,ai.vespa.validation;version\=1.0.0,com.yahoo.binaryprefix;version\=1.0.0,com.yahoo.collections;version\=1.0.0,com.yahoo.compress;version\=1.0.0,com.yahoo.concurrent.classlock;version\=1.0.0,com.yahoo.concurrent.maintenance;version\=1.0.0,com.yahoo.concurrent;version\=1.0.0,com.yahoo.data.access.simple;version\=1.0.0,com.yahoo.data.access.slime;version\=1.0.0,com.yahoo.data.access;version\=1.0.0,com.yahoo.errorhandling;version\=1.0.0,com.yahoo.exception;version\=1.0.0,com.yahoo.geo;version\=1.0.0,com.yahoo.io.reader;version\=1.0.0,com.yahoo.io;version\=1.0.0,com.yahoo.javacc;version\=1.0.0,com.yahoo.lang;version\=1.0.0,com.yahoo.nativec;version\=1.0.0,com.yahoo.net;version\=1.0.0,com.yahoo.path;version\=1.0.0,com.yahoo.protect;version\=1.0.0,com.yahoo.reflection;version\=1.0.0,com.yahoo.slime;version\=1.0.0,com.yahoo.stream;version\=1.0.0,com.yahoo.system.execution;version\=1.0.0,com.yahoo.system;version\=1.0.0,com.yahoo.tensor.evaluation;version\=1.0.0,com.yahoo.tensor.functions;version\=1.0.0,com.yahoo.tensor.serialization;version\=1.0.0,com.yahoo.tensor;version\=1.0.0,com.yahoo.text.internal;version\=1.0.0,com.yahoo.text;version\=1.0.0,com.yahoo.time;version\=1.0.0,com.yahoo.transaction;version\=1.0.0,com.yahoo.vespa.objects;version\=1.0.0,com.yahoo.yolean.chain;version\=1.0.0,com.yahoo.yolean.concurrent;version\=1.0.0,com.yahoo.yolean.function;version\=1.0.0,com.yahoo.yolean.system;version\=1.0.0,com.yahoo.yolean.trace;version\=1.0.0,com.yahoo.yolean;version\=1.0.0, com.yahoo.log.event;version\=1.0.0,com.yahoo.log.impl;version\=1.0.0,com.yahoo.log;version\=1.0.0, javax.xml.bind;version\="2.3";uses\:\="javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.annotation;version\="2.3";uses\:\="javax.xml.bind,javax.xml.parsers,javax.xml.transform,javax.xml.transform.dom,org.w3c.dom",javax.xml.bind.annotation.adapters;version\="2.3",javax.xml.bind.attachment;version\="2.3";uses\:\="javax.activation",javax.xml.bind.helpers;version\="2.3";uses\:\="javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.util;version\="2.3";uses\:\="javax.xml.bind,javax.xml.transform.sax", com.sun.istack;version\="3.0.5";uses\:\="javax.activation,javax.xml.stream,org.xml.sax,org.xml.sax.helpers",com.sun.istack.localization;version\="3.0.5",com.sun.istack.logging;version\="3.0.5",com.sun.xml.bind;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.annotation;version\="2.3.0",com.sun.xml.bind.api;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.api.impl;version\="2.3.0",com.sun.xml.bind.marshaller;uses\:\="javax.xml.parsers,org.w3c.dom,org.xml.sax,org.xml.sax.helpers";version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;version\="2.3.0",com.sun.xml.bind.v2;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.core;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.impl,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.namespace,javax.xml.transform";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.nav";version\="2.3.0",com.sun.xml.bind.v2.model.nav;uses\:\="com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.util;uses\:\="javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.xml.bind.v2.model.annotation,javax.activation,javax.xml.bind,javax.xml.bind.annotation.adapters";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen.episode;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="javax.xml.parsers,javax.xml.transform,javax.xml.validation,javax.xml.xpath";version\="2.3.0",com.sun.xml.txw2;uses\:\="com.sun.xml.txw2.output,javax.xml.namespace";version\="2.3.0",com.sun.xml.txw2.annotation;version\="2.3.0",com.sun.xml.txw2.output;uses\:\="com.sun.xml.txw2,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.dom,javax.xml.transform.sax,javax.xml.transform.stream,org.w3c.dom,org.xml.sax,org.xml.sax.ext,org.xml.sax.helpers";version\="2.3.0", com.sun.xml.bind;uses\:\="com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.datatype,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.api;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.xml.bind,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.marshaller;version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;uses\:\="com.sun.xml.bind,javax.xml.bind.helpers,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,javax.xml.bind";version\="2.3.0",com.sun.xml.bind.v2.bytecode;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.model.runtime;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.namespace,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.istack,com.sun.xml.bind.api,com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.property,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.output;uses\:\="com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.runtime,com.sun.xml.fastinfoset.stax,javax.xml.stream,org.jvnet.staxex,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.property;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,com.sun.xml.bind.v2.runtime.unmarshaller,com.sun.xml.bind.v2.util,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect.opt;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="com.sun.xml.bind,com.sun.xml.bind.api,com.sun.xml.bind.unmarshaller,com.sun.xml.bind.util,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.reflect,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.txw2.output,javax.xml.bind,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.schemagen.xmlschema;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.namespace,javax.xml.transform.stream,org.xml.sax";version\="2.3.0", javax.activation;uses\:\="com.sun.activation.registries";version\="1.2",com.sun.activation.viewers;uses\:\="javax.activation";version\="1.2.0",com.sun.activation.registries;version\="1.2.0" +exportPackages=org.osgi.framework; version\="1.10.0", org.osgi.framework.connect; version\="1.0.0", org.osgi.framework.dto; uses\:\="org.osgi.dto"; version\="1.8.0", org.osgi.framework.hooks.bundle; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.resolver; uses\:\="org.osgi.framework.wiring"; version\="1.0.0", org.osgi.framework.hooks.service; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.weaving; uses\:\="org.osgi.framework.wiring"; version\="1.1.0", org.osgi.framework.launch; uses\:\="org.osgi.framework"; version\="1.2.0", org.osgi.framework.namespace; uses\:\="org.osgi.resource"; version\="1.2.0", org.osgi.framework.startlevel; uses\:\="org.osgi.framework"; version\="1.0.0", org.osgi.framework.startlevel.dto; uses\:\="org.osgi.dto"; version\="1.0.0", org.osgi.framework.wiring; uses\:\="org.osgi.framework,org.osgi.resource"; version\="1.2.0", org.osgi.framework.wiring.dto; uses\:\="org.osgi.dto,org.osgi.resource.dto"; version\="1.3.0", org.osgi.resource; version\="1.0.1", org.osgi.resource.dto; uses\:\="org.osgi.dto"; version\="1.0.1", org.osgi.service.packageadmin; uses\:\="org.osgi.framework"; version\="1.2.1", org.osgi.service.startlevel; uses\:\="org.osgi.framework"; version\="1.1.1", org.osgi.service.url; version\="1.0.1", org.osgi.service.resolver; uses\:\="org.osgi.resource"; version\="1.1.1", org.osgi.util.tracker; uses\:\="org.osgi.framework"; version\="1.5.3", org.osgi.dto; version\="1.1.1", org.osgi.service.condition; version\="1.0.0", javax.security.auth.callback; version\="0.0.0.JavaSE_017", javax.security.auth.login; version\="0.0.0.JavaSE_017", javax.net.ssl; version\="0.0.0.JavaSE_017", java.lang.constant; version\="0.0.0.JavaSE_017", java.security.interfaces; version\="0.0.0.JavaSE_017", java.text.spi; version\="0.0.0.JavaSE_017", java.nio.channels.spi; version\="0.0.0.JavaSE_017", java.math; version\="0.0.0.JavaSE_017", java.nio.file; version\="0.0.0.JavaSE_017", java.util.concurrent.atomic; version\="0.0.0.JavaSE_017", java.security.cert; version\="0.0.0.JavaSE_017", java.security.spec; version\="0.0.0.JavaSE_017", java.nio.channels; version\="0.0.0.JavaSE_017", java.time.chrono; version\="0.0.0.JavaSE_017", javax.crypto; version\="0.0.0.JavaSE_017", java.time.zone; version\="0.0.0.JavaSE_017", java.nio.charset; version\="0.0.0.JavaSE_017", java.io; version\="0.0.0.JavaSE_017", java.util.spi; version\="0.0.0.JavaSE_017", java.net; version\="0.0.0.JavaSE_017", javax.security.cert; version\="0.0.0.JavaSE_017", java.lang.annotation; version\="0.0.0.JavaSE_017", javax.security.auth.spi; version\="0.0.0.JavaSE_017", java.util.concurrent; version\="0.0.0.JavaSE_017", java.nio.charset.spi; version\="0.0.0.JavaSE_017", javax.crypto.interfaces; version\="0.0.0.JavaSE_017", java.util; version\="0.0.0.JavaSE_017", java.security; version\="0.0.0.JavaSE_017", java.nio.file.spi; version\="0.0.0.JavaSE_017", java.nio; version\="0.0.0.JavaSE_017", java.util.jar; version\="0.0.0.JavaSE_017", javax.security.auth; version\="0.0.0.JavaSE_017", java.lang.ref; version\="0.0.0.JavaSE_017", java.util.regex; version\="0.0.0.JavaSE_017", java.net.spi; version\="0.0.0.JavaSE_017", java.lang.module; version\="0.0.0.JavaSE_017", java.lang.invoke; version\="0.0.0.JavaSE_017", java.time.format; version\="0.0.0.JavaSE_017", java.util.concurrent.locks; version\="0.0.0.JavaSE_017", java.time.temporal; version\="0.0.0.JavaSE_017", java.util.zip; version\="0.0.0.JavaSE_017", java.nio.file.attribute; version\="0.0.0.JavaSE_017", java.util.random; version\="0.0.0.JavaSE_017", java.text; version\="0.0.0.JavaSE_017", javax.crypto.spec; version\="0.0.0.JavaSE_017", java.util.stream; version\="0.0.0.JavaSE_017", java.time; version\="0.0.0.JavaSE_017", java.lang; version\="0.0.0.JavaSE_017", java.lang.runtime; version\="0.0.0.JavaSE_017", java.util.function; version\="0.0.0.JavaSE_017", javax.net; version\="0.0.0.JavaSE_017", javax.security.auth.x500; version\="0.0.0.JavaSE_017", java.lang.reflect; version\="0.0.0.JavaSE_017", javax.lang.model; version\="0.0.0.JavaSE_017", javax.annotation.processing; version\="0.0.0.JavaSE_017", javax.lang.model.element; version\="0.0.0.JavaSE_017", javax.tools; version\="0.0.0.JavaSE_017", javax.lang.model.type; version\="0.0.0.JavaSE_017", javax.lang.model.util; version\="0.0.0.JavaSE_017", java.awt.datatransfer; version\="0.0.0.JavaSE_017", java.awt.desktop; version\="0.0.0.JavaSE_017", javax.swing.plaf.synth; version\="0.0.0.JavaSE_017", java.beans; version\="0.0.0.JavaSE_017", javax.swing.text.html.parser; version\="0.0.0.JavaSE_017", javax.swing.text.rtf; version\="0.0.0.JavaSE_017", java.awt.font; version\="0.0.0.JavaSE_017", javax.imageio; version\="0.0.0.JavaSE_017", java.awt.im.spi; version\="0.0.0.JavaSE_017", java.applet; version\="0.0.0.JavaSE_017", javax.sound.midi; version\="0.0.0.JavaSE_017", java.awt.dnd; version\="0.0.0.JavaSE_017", javax.swing.text; version\="0.0.0.JavaSE_017", javax.swing.plaf.basic; version\="0.0.0.JavaSE_017", javax.swing.undo; version\="0.0.0.JavaSE_017", javax.swing.plaf; version\="0.0.0.JavaSE_017", javax.swing.filechooser; version\="0.0.0.JavaSE_017", javax.imageio.event; version\="0.0.0.JavaSE_017", javax.sound.sampled; version\="0.0.0.JavaSE_017", javax.print.attribute; version\="0.0.0.JavaSE_017", javax.print; version\="0.0.0.JavaSE_017", javax.swing.plaf.nimbus; version\="0.0.0.JavaSE_017", javax.accessibility; version\="0.0.0.JavaSE_017", java.awt.event; version\="0.0.0.JavaSE_017", javax.swing.text.html; version\="0.0.0.JavaSE_017", javax.imageio.spi; version\="0.0.0.JavaSE_017", javax.swing.border; version\="0.0.0.JavaSE_017", javax.sound.sampled.spi; version\="0.0.0.JavaSE_017", javax.imageio.plugins.bmp; version\="0.0.0.JavaSE_017", java.awt.color; version\="0.0.0.JavaSE_017", java.awt.geom; version\="0.0.0.JavaSE_017", javax.imageio.plugins.jpeg; version\="0.0.0.JavaSE_017", javax.swing.tree; version\="0.0.0.JavaSE_017", javax.imageio.plugins.tiff; version\="0.0.0.JavaSE_017", java.awt.print; version\="0.0.0.JavaSE_017", java.awt.image; version\="0.0.0.JavaSE_017", javax.imageio.metadata; version\="0.0.0.JavaSE_017", javax.swing.table; version\="0.0.0.JavaSE_017", javax.sound.midi.spi; version\="0.0.0.JavaSE_017", javax.print.attribute.standard; version\="0.0.0.JavaSE_017", javax.swing.colorchooser; version\="0.0.0.JavaSE_017", javax.swing; version\="0.0.0.JavaSE_017", java.awt.image.renderable; version\="0.0.0.JavaSE_017", javax.swing.plaf.multi; version\="0.0.0.JavaSE_017", java.awt.im; version\="0.0.0.JavaSE_017", javax.print.event; version\="0.0.0.JavaSE_017", javax.swing.plaf.metal; version\="0.0.0.JavaSE_017", java.beans.beancontext; version\="0.0.0.JavaSE_017", java.awt; version\="0.0.0.JavaSE_017", javax.imageio.stream; version\="0.0.0.JavaSE_017", javax.swing.event; version\="0.0.0.JavaSE_017", java.lang.instrument; version\="0.0.0.JavaSE_017", java.util.logging; version\="0.0.0.JavaSE_017", javax.management.timer; version\="0.0.0.JavaSE_017", javax.management; version\="0.0.0.JavaSE_017", javax.management.relation; version\="0.0.0.JavaSE_017", javax.management.loading; version\="0.0.0.JavaSE_017", javax.management.openmbean; version\="0.0.0.JavaSE_017", java.lang.management; version\="0.0.0.JavaSE_017", javax.management.remote; version\="0.0.0.JavaSE_017", javax.management.monitor; version\="0.0.0.JavaSE_017", javax.management.modelmbean; version\="0.0.0.JavaSE_017", javax.management.remote.rmi; version\="0.0.0.JavaSE_017", javax.naming.event; version\="0.0.0.JavaSE_017", javax.naming.ldap.spi; version\="0.0.0.JavaSE_017", javax.naming; version\="0.0.0.JavaSE_017", javax.naming.spi; version\="0.0.0.JavaSE_017", javax.naming.ldap; version\="0.0.0.JavaSE_017", javax.naming.directory; version\="0.0.0.JavaSE_017", java.net.http; version\="0.0.0.JavaSE_017", java.util.prefs; version\="0.0.0.JavaSE_017", java.rmi.registry; version\="0.0.0.JavaSE_017", javax.rmi.ssl; version\="0.0.0.JavaSE_017", java.rmi.dgc; version\="0.0.0.JavaSE_017", java.rmi; version\="0.0.0.JavaSE_017", java.rmi.server; version\="0.0.0.JavaSE_017", javax.script; version\="0.0.0.JavaSE_017", org.ietf.jgss; version\="0.0.0.JavaSE_017", javax.security.auth.kerberos; version\="0.0.0.JavaSE_017", javax.security.sasl; version\="0.0.0.JavaSE_017", javax.smartcardio; version\="0.0.0.JavaSE_017", java.sql; version\="0.0.0.JavaSE_017", javax.sql; version\="0.0.0.JavaSE_017", javax.sql.rowset.spi; version\="0.0.0.JavaSE_017", javax.sql.rowset.serial; version\="0.0.0.JavaSE_017", javax.sql.rowset; version\="0.0.0.JavaSE_017", javax.transaction.xa; version\="0.0.0.JavaSE_017", javax.xml; version\="0.0.0.JavaSE_017", javax.xml.transform.sax; version\="0.0.0.JavaSE_017", javax.xml.datatype; version\="0.0.0.JavaSE_017", javax.xml.catalog; version\="0.0.0.JavaSE_017", org.w3c.dom.traversal; version\="0.0.0.JavaSE_017", javax.xml.stream.events; version\="0.0.0.JavaSE_017", javax.xml.stream; version\="0.0.0.JavaSE_017", org.xml.sax; version\="0.0.0.JavaSE_017", javax.xml.transform; version\="0.0.0.JavaSE_017", javax.xml.xpath; version\="0.0.0.JavaSE_017", org.w3c.dom.events; version\="0.0.0.JavaSE_017", org.w3c.dom.ranges; version\="0.0.0.JavaSE_017", org.w3c.dom.ls; version\="0.0.0.JavaSE_017", javax.xml.stream.util; version\="0.0.0.JavaSE_017", javax.xml.namespace; version\="0.0.0.JavaSE_017", javax.xml.transform.stax; version\="0.0.0.JavaSE_017", org.xml.sax.helpers; version\="0.0.0.JavaSE_017", org.w3c.dom.views; version\="0.0.0.JavaSE_017", org.w3c.dom.bootstrap; version\="0.0.0.JavaSE_017", org.w3c.dom; version\="0.0.0.JavaSE_017", javax.xml.transform.stream; version\="0.0.0.JavaSE_017", javax.xml.transform.dom; version\="0.0.0.JavaSE_017", javax.xml.validation; version\="0.0.0.JavaSE_017", javax.xml.parsers; version\="0.0.0.JavaSE_017", org.xml.sax.ext; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.spec; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.keyinfo; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig; version\="0.0.0.JavaSE_017", com.sun.java.accessibility.util; version\="0.0.0.JavaSE_017", com.sun.tools.attach.spi; version\="0.0.0.JavaSE_017", com.sun.tools.attach; version\="0.0.0.JavaSE_017", com.sun.source.doctree; version\="0.0.0.JavaSE_017", com.sun.source.tree; version\="0.0.0.JavaSE_017", com.sun.source.util; version\="0.0.0.JavaSE_017", com.sun.tools.javac; version\="0.0.0.JavaSE_017", jdk.dynalink.beans; version\="0.0.0.JavaSE_017", jdk.dynalink.linker.support; version\="0.0.0.JavaSE_017", jdk.dynalink.support; version\="0.0.0.JavaSE_017", jdk.dynalink; version\="0.0.0.JavaSE_017", jdk.dynalink.linker; version\="0.0.0.JavaSE_017", com.sun.net.httpserver; version\="0.0.0.JavaSE_017", com.sun.net.httpserver.spi; version\="0.0.0.JavaSE_017", com.sun.jarsigner; version\="0.0.0.JavaSE_017", jdk.security.jarsigner; version\="0.0.0.JavaSE_017", jdk.javadoc.doclet; version\="0.0.0.JavaSE_017", com.sun.tools.jconsole; version\="0.0.0.JavaSE_017", com.sun.jdi.connect; version\="0.0.0.JavaSE_017", com.sun.jdi.event; version\="0.0.0.JavaSE_017", com.sun.jdi.connect.spi; version\="0.0.0.JavaSE_017", com.sun.jdi; version\="0.0.0.JavaSE_017", com.sun.jdi.request; version\="0.0.0.JavaSE_017", jdk.jfr.consumer; version\="0.0.0.JavaSE_017", jdk.jfr; version\="0.0.0.JavaSE_017", jdk.jshell; version\="0.0.0.JavaSE_017", jdk.jshell.execution; version\="0.0.0.JavaSE_017", jdk.jshell.spi; version\="0.0.0.JavaSE_017", jdk.jshell.tool; version\="0.0.0.JavaSE_017", netscape.javascript; version\="0.0.0.JavaSE_017", com.sun.management; version\="0.0.0.JavaSE_017", jdk.management.jfr; version\="0.0.0.JavaSE_017", jdk.net; version\="0.0.0.JavaSE_017", jdk.nio; version\="0.0.0.JavaSE_017", jdk.nio.mapmode; version\="0.0.0.JavaSE_017", com.sun.nio.sctp; version\="0.0.0.JavaSE_017", com.sun.security.auth.module; version\="0.0.0.JavaSE_017", com.sun.security.auth.login; version\="0.0.0.JavaSE_017", com.sun.security.auth; version\="0.0.0.JavaSE_017", com.sun.security.auth.callback; version\="0.0.0.JavaSE_017", com.sun.security.jgss; version\="0.0.0.JavaSE_017", sun.reflect; version\="0.0.0.JavaSE_017", sun.misc; version\="0.0.0.JavaSE_017", com.sun.nio.file; version\="0.0.0.JavaSE_017", jdk.swing.interop; version\="0.0.0.JavaSE_017", org.w3c.dom.stylesheets; version\="0.0.0.JavaSE_017", org.w3c.dom.html; version\="0.0.0.JavaSE_017", org.w3c.dom.xpath; version\="0.0.0.JavaSE_017", org.w3c.dom.css; version\="0.0.0.JavaSE_017", com.yahoo.jdisc, com.yahoo.jdisc.application, com.yahoo.jdisc.handler, com.yahoo.jdisc.service, com.yahoo.jdisc.statistics, com.yahoo.jdisc.refcount, javax.inject;version\=1.0.0, org.aopalliance.intercept, org.aopalliance.aop, com.google.common.annotations;version\="27.1.0",com.google.common.base;version\="27.1.0",com.google.common.cache;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent",com.google.common.collect;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.escape;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.eventbus;version\="27.1.0",com.google.common.graph;version\="27.1.0";uses\:\="com.google.common.collect",com.google.common.hash;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.html;version\="27.1.0";uses\:\="com.google.common.escape",com.google.common.io;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.graph,com.google.common.hash",com.google.common.math;version\="27.1.0",com.google.common.net;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.escape",com.google.common.primitives;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.reflect;version\="27.1.0";uses\:\="com.google.common.collect,com.google.common.io",com.google.common.util.concurrent;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent.internal",com.google.common.xml;version\="27.1.0";uses\:\="com.google.common.escape", com.google.inject;version\="1.4",com.google.inject.binder;version\="1.4",com.google.inject.matcher;version\="1.4",com.google.inject.multibindings;version\="1.4",com.google.inject.name;version\="1.4",com.google.inject.spi;version\="1.4",com.google.inject.util;version\="1.4", org.slf4j;version\=1.7.32, org.slf4j.spi;version\=1.7.32, org.slf4j.helpers;version\=1.7.32, org.slf4j.event;version\=1.7.32, org.slf4j.impl;version\=1.7.32, org.apache.commons.logging;version\=1.2, org.apache.commons.logging.impl;version\=1.2, org.apache.log4j;version\=1.2.17,org.apache.log4j.helpers;version\=1.2.17,org.apache.log4j.spi;version\=1.2.17,org.apache.log4j.xml;version\=1.2.17, com.yahoo.component.annotation;version\="1.0.0", com.yahoo.config;version\=1.0.0, com.yahoo.vespa.defaults;version\=1.0.0, ai.vespa.http;version\=1.0.0,ai.vespa.validation;version\=1.0.0,com.yahoo.binaryprefix;version\=1.0.0,com.yahoo.collections;version\=1.0.0,com.yahoo.compress;version\=1.0.0,com.yahoo.concurrent.classlock;version\=1.0.0,com.yahoo.concurrent.maintenance;version\=1.0.0,com.yahoo.concurrent;version\=1.0.0,com.yahoo.data.access.simple;version\=1.0.0,com.yahoo.data.access.slime;version\=1.0.0,com.yahoo.data.access;version\=1.0.0,com.yahoo.errorhandling;version\=1.0.0,com.yahoo.exception;version\=1.0.0,com.yahoo.geo;version\=1.0.0,com.yahoo.io.reader;version\=1.0.0,com.yahoo.io;version\=1.0.0,com.yahoo.javacc;version\=1.0.0,com.yahoo.lang;version\=1.0.0,com.yahoo.nativec;version\=1.0.0,com.yahoo.net;version\=1.0.0,com.yahoo.path;version\=1.0.0,com.yahoo.protect;version\=1.0.0,com.yahoo.reflection;version\=1.0.0,com.yahoo.slime;version\=1.0.0,com.yahoo.stream;version\=1.0.0,com.yahoo.system.execution;version\=1.0.0,com.yahoo.system;version\=1.0.0,com.yahoo.tensor.evaluation;version\=1.0.0,com.yahoo.tensor.functions;version\=1.0.0,com.yahoo.tensor.serialization;version\=1.0.0,com.yahoo.tensor;version\=1.0.0,com.yahoo.text.internal;version\=1.0.0,com.yahoo.text;version\=1.0.0,com.yahoo.time;version\=1.0.0,com.yahoo.transaction;version\=1.0.0,com.yahoo.vespa.objects;version\=1.0.0,com.yahoo.yolean.chain;version\=1.0.0,com.yahoo.yolean.concurrent;version\=1.0.0,com.yahoo.yolean.function;version\=1.0.0,com.yahoo.yolean.system;version\=1.0.0,com.yahoo.yolean.trace;version\=1.0.0,com.yahoo.yolean;version\=1.0.0, com.yahoo.log.event;version\=1.0.0,com.yahoo.log.impl;version\=1.0.0,com.yahoo.log;version\=1.0.0, javax.xml.bind;version\="2.3";uses\:\="javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.annotation;version\="2.3";uses\:\="javax.xml.bind,javax.xml.parsers,javax.xml.transform,javax.xml.transform.dom,org.w3c.dom",javax.xml.bind.annotation.adapters;version\="2.3",javax.xml.bind.attachment;version\="2.3";uses\:\="javax.activation",javax.xml.bind.helpers;version\="2.3";uses\:\="javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.util;version\="2.3";uses\:\="javax.xml.bind,javax.xml.transform.sax", com.sun.istack;version\="3.0.5";uses\:\="javax.activation,javax.xml.stream,org.xml.sax,org.xml.sax.helpers",com.sun.istack.localization;version\="3.0.5",com.sun.istack.logging;version\="3.0.5",com.sun.xml.bind;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.annotation;version\="2.3.0",com.sun.xml.bind.api;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.api.impl;version\="2.3.0",com.sun.xml.bind.marshaller;uses\:\="javax.xml.parsers,org.w3c.dom,org.xml.sax,org.xml.sax.helpers";version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;version\="2.3.0",com.sun.xml.bind.v2;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.core;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.impl,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.namespace,javax.xml.transform";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.nav";version\="2.3.0",com.sun.xml.bind.v2.model.nav;uses\:\="com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.util;uses\:\="javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.xml.bind.v2.model.annotation,javax.activation,javax.xml.bind,javax.xml.bind.annotation.adapters";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen.episode;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="javax.xml.parsers,javax.xml.transform,javax.xml.validation,javax.xml.xpath";version\="2.3.0",com.sun.xml.txw2;uses\:\="com.sun.xml.txw2.output,javax.xml.namespace";version\="2.3.0",com.sun.xml.txw2.annotation;version\="2.3.0",com.sun.xml.txw2.output;uses\:\="com.sun.xml.txw2,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.dom,javax.xml.transform.sax,javax.xml.transform.stream,org.w3c.dom,org.xml.sax,org.xml.sax.ext,org.xml.sax.helpers";version\="2.3.0", com.sun.xml.bind;uses\:\="com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.datatype,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.api;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.xml.bind,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.marshaller;version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;uses\:\="com.sun.xml.bind,javax.xml.bind.helpers,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,javax.xml.bind";version\="2.3.0",com.sun.xml.bind.v2.bytecode;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.model.runtime;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.namespace,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.istack,com.sun.xml.bind.api,com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.property,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.output;uses\:\="com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.runtime,com.sun.xml.fastinfoset.stax,javax.xml.stream,org.jvnet.staxex,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.property;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,com.sun.xml.bind.v2.runtime.unmarshaller,com.sun.xml.bind.v2.util,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect.opt;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="com.sun.xml.bind,com.sun.xml.bind.api,com.sun.xml.bind.unmarshaller,com.sun.xml.bind.util,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.reflect,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.txw2.output,javax.xml.bind,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.schemagen.xmlschema;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.namespace,javax.xml.transform.stream,org.xml.sax";version\="2.3.0", javax.activation;uses\:\="com.sun.activation.registries";version\="1.2",com.sun.activation.viewers;uses\:\="javax.activation";version\="1.2.0",com.sun.activation.registries;version\="1.2.0" diff --git a/jrt/src/com/yahoo/jrt/MandatoryMethods.java b/jrt/src/com/yahoo/jrt/MandatoryMethods.java index b1355c0fb1e..a73e2bfc6dd 100644 --- a/jrt/src/com/yahoo/jrt/MandatoryMethods.java +++ b/jrt/src/com/yahoo/jrt/MandatoryMethods.java @@ -23,6 +23,7 @@ class MandatoryMethods { parent.addMethod(m); //--------------------------------------------------------------------- m = new Method("frt.rpc.getMethodList", "", "SSS", this::getMethodList); + m.requireCapabilities(CapabilitySet.none()); m.methodDesc("Obtain a list of all available methods"); m.returnDesc(0, "names", "Method names"); m.returnDesc(1, "params", "Method parameter types"); @@ -30,6 +31,7 @@ class MandatoryMethods { parent.addMethod(m); //--------------------------------------------------------------------- m = new Method("frt.rpc.getMethodInfo", "s", "sssSSSS", this::getMethodInfo); + m.requireCapabilities(CapabilitySet.none()); m.methodDesc("Obtain detailed information about a single method"); m.paramDesc (0, "methodName", "The method we want information about"); m.returnDesc(0, "desc", "Description of what the method does"); diff --git a/jrt_test/CMakeLists.txt b/jrt_test/CMakeLists.txt index ea8c8b94faa..a678cbf112a 100644 --- a/jrt_test/CMakeLists.txt +++ b/jrt_test/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib fnet diff --git a/jrt_test/src/tests/echo/echo-client.cpp b/jrt_test/src/tests/echo/echo-client.cpp index a7c8d309114..16a4a2877b2 100644 --- a/jrt_test/src/tests/echo/echo-client.cpp +++ b/jrt_test/src/tests/echo/echo-client.cpp @@ -79,8 +79,8 @@ public: } else { printf("Return values != parameters.\n"); } - req->SubRef(); - target->SubRef(); + req->internal_subref(); + target->internal_subref(); return 0; } }; diff --git a/jrt_test/src/tests/mandatory-methods/extract-reflection.cpp b/jrt_test/src/tests/mandatory-methods/extract-reflection.cpp index cdcf02da465..fae7bc90abb 100644 --- a/jrt_test/src/tests/mandatory-methods/extract-reflection.cpp +++ b/jrt_test/src/tests/mandatory-methods/extract-reflection.cpp @@ -14,16 +14,16 @@ public: void GetReq(FRT_RPCRequest **req, FRT_Supervisor *supervisor) { if ((*req) != nullptr) - (*req)->SubRef(); + (*req)->internal_subref(); (*req) = supervisor->AllocRPCRequest(); } void FreeReqs(FRT_RPCRequest *r1, FRT_RPCRequest *r2) { if (r1 != nullptr) - r1->SubRef(); + r1->internal_subref(); if (r2 != nullptr) - r2->SubRef(); + r2->internal_subref(); } void DumpMethodInfo(const char *indent, FRT_RPCRequest *info, @@ -90,7 +90,7 @@ public: break; } std::this_thread::sleep_for(1s); - target->SubRef(); + target->internal_subref(); target = supervisor.GetTarget(argv[1]); } if (info->IsError()) { @@ -133,7 +133,7 @@ public: m_list->GetErrorMessage()); } FreeReqs(m_list, info); - target->SubRef(); + target->internal_subref(); return 0; } }; diff --git a/jrt_test/src/tests/rpc-error/test-errors.cpp b/jrt_test/src/tests/rpc-error/test-errors.cpp index e64c2abfff6..1683d1e11f8 100644 --- a/jrt_test/src/tests/rpc-error/test-errors.cpp +++ b/jrt_test/src/tests/rpc-error/test-errors.cpp @@ -20,7 +20,7 @@ public: } void fini() { - target->SubRef(); + target->internal_subref(); target = nullptr; client = nullptr; } @@ -51,7 +51,7 @@ TestErrors::testNoError() } else { EXPECT_TRUE(false); } - req1->SubRef(); + req1->internal_subref(); } @@ -64,7 +64,7 @@ TestErrors::testNoSuchMethod() EXPECT_TRUE(req1->IsError()); EXPECT_TRUE(0 == req1->GetReturn()->GetNumValues()); EXPECT_TRUE(FRTE_RPC_NO_SUCH_METHOD == req1->GetErrorCode()); - req1->SubRef(); + req1->internal_subref(); } @@ -80,7 +80,7 @@ TestErrors::testWrongParameters() EXPECT_TRUE(req1->IsError()); EXPECT_TRUE(0 == req1->GetReturn()->GetNumValues()); EXPECT_TRUE(FRTE_RPC_WRONG_PARAMS == req1->GetErrorCode()); - req1->SubRef(); + req1->internal_subref(); FRT_RPCRequest *req2 = client->AllocRPCRequest(); req2->SetMethodName("test"); @@ -90,7 +90,7 @@ TestErrors::testWrongParameters() EXPECT_TRUE(req2->IsError()); EXPECT_TRUE(0 == req2->GetReturn()->GetNumValues()); EXPECT_TRUE(FRTE_RPC_WRONG_PARAMS == req2->GetErrorCode()); - req2->SubRef(); + req2->internal_subref(); FRT_RPCRequest *req3 = client->AllocRPCRequest(); req3->SetMethodName("test"); @@ -102,7 +102,7 @@ TestErrors::testWrongParameters() EXPECT_TRUE(req3->IsError()); EXPECT_TRUE(0 == req3->GetReturn()->GetNumValues()); EXPECT_TRUE(FRTE_RPC_WRONG_PARAMS == req3->GetErrorCode()); - req3->SubRef(); + req3->internal_subref(); } @@ -118,7 +118,7 @@ TestErrors::testWrongReturnValues() EXPECT_TRUE(req1->IsError()); EXPECT_TRUE(0 == req1->GetReturn()->GetNumValues()); EXPECT_TRUE(FRTE_RPC_WRONG_RETURN == req1->GetErrorCode()); - req1->SubRef(); + req1->internal_subref(); } @@ -134,7 +134,7 @@ TestErrors::testMethodFailed() EXPECT_TRUE(req1->IsError()); EXPECT_TRUE(0 == req1->GetReturn()->GetNumValues()); EXPECT_TRUE(75000 == req1->GetErrorCode()); - req1->SubRef(); + req1->internal_subref(); FRT_RPCRequest *req2 = client->AllocRPCRequest(); req2->SetMethodName("test"); @@ -145,7 +145,7 @@ TestErrors::testMethodFailed() EXPECT_TRUE(req2->IsError()); EXPECT_TRUE(0 == req2->GetReturn()->GetNumValues()); EXPECT_TRUE(75000 == req2->GetErrorCode()); - req2->SubRef(); + req2->internal_subref(); } diff --git a/linguistics/src/main/java/com/yahoo/language/Linguistics.java b/linguistics/src/main/java/com/yahoo/language/Linguistics.java index 791bc5dabcf..6fa63e657bd 100644 --- a/linguistics/src/main/java/com/yahoo/language/Linguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/Linguistics.java @@ -47,7 +47,7 @@ public interface Linguistics { /** * Returns a thread-unsafe tokenizer. - * This is used at indexing time to produce a optionally stemmed and + * This is used at indexing time to produce an optionally stemmed and * transformed (accent normalized) stream of indexable tokens. */ Tokenizer getTokenizer(); diff --git a/logd/CMakeLists.txt b/logd/CMakeLists.txt index d02b99a393a..5823ebf54a7 100644 --- a/logd/CMakeLists.txt +++ b/logd/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib config_cloudconfig diff --git a/logd/src/logd/rpc_forwarder.cpp b/logd/src/logd/rpc_forwarder.cpp index fbd39afeba4..b89eb87e2f7 100644 --- a/logd/src/logd/rpc_forwarder.cpp +++ b/logd/src/logd/rpc_forwarder.cpp @@ -31,7 +31,7 @@ public: : _request(new FRT_RPCRequest()) {} ~GuardedRequest() { - _request->SubRef(); + _request->internal_subref(); } FRT_RPCRequest& operator*() const { return *_request; } FRT_RPCRequest* get() const { return _request; } diff --git a/logd/src/logd/rpc_forwarder.h b/logd/src/logd/rpc_forwarder.h index 75d41259eda..864c7b666cb 100644 --- a/logd/src/logd/rpc_forwarder.h +++ b/logd/src/logd/rpc_forwarder.h @@ -17,7 +17,7 @@ struct Metrics; struct RpcTargetSubRef { void operator()(FRT_Target* target) const noexcept { - target->SubRef(); + target->internal_subref(); } }; using RpcTargetGuard = std::unique_ptr<FRT_Target, RpcTargetSubRef>; diff --git a/logd/src/logd/watcher.cpp b/logd/src/logd/watcher.cpp index c3752fcc3e8..af1efc3077d 100644 --- a/logd/src/logd/watcher.cpp +++ b/logd/src/logd/watcher.cpp @@ -8,6 +8,7 @@ #include <vespa/vespalib/util/time.h> #include <vespa/vespalib/util/size_literals.h> #include <thread> +#include <cinttypes> #include <fcntl.h> #include <glob.h> #include <sys/stat.h> diff --git a/lowercasing_test/CMakeLists.txt b/lowercasing_test/CMakeLists.txt index b08a6ef350d..119209d4227 100644 --- a/lowercasing_test/CMakeLists.txt +++ b/lowercasing_test/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib searchlib diff --git a/maven-plugins/allowed-maven-dependencies.txt b/maven-plugins/allowed-maven-dependencies.txt index a3cc4cc7045..5272dd21a31 100644 --- a/maven-plugins/allowed-maven-dependencies.txt +++ b/maven-plugins/allowed-maven-dependencies.txt @@ -3,9 +3,9 @@ #[non-test] # Contains dependencies that are not used exclusively in 'test' scope aopalliance:aopalliance:1.0 -com.fasterxml.jackson.core:jackson-annotations:2.13.4 -com.fasterxml.jackson.core:jackson-core:2.13.4 -com.fasterxml.jackson.core:jackson-databind:2.13.4.2 +com.fasterxml.jackson.core:jackson-annotations:2.14.2 +com.fasterxml.jackson.core:jackson-core:2.14.2 +com.fasterxml.jackson.core:jackson-databind:2.14.2 com.google.errorprone:error_prone_annotations:2.18.0 com.google.guava:failureaccess:1.0.1 com.google.guava:guava:27.1-jre diff --git a/messagebus/CMakeLists.txt b/messagebus/CMakeLists.txt index 30e795cac1d..ab37173a5ea 100644 --- a/messagebus/CMakeLists.txt +++ b/messagebus/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog config_cloudconfig vespalib diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java index b4fa7d8f887..6afc2039c38 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java @@ -29,6 +29,7 @@ import com.yahoo.messagebus.network.NetworkOwner; import com.yahoo.messagebus.routing.Hop; import com.yahoo.messagebus.routing.Route; import com.yahoo.messagebus.routing.RoutingNode; +import com.yahoo.security.tls.CapabilitySet; import java.io.PrintWriter; import java.io.StringWriter; @@ -100,6 +101,7 @@ public class RPCNetwork implements Network, MethodHandler { servicePool = new RPCServicePool(this, 4096); Method method = new Method("mbus.getVersion", "", "s", this); + method.requireCapabilities(CapabilitySet.none()); method.methodDesc("Retrieves the message bus version."); method.returnDesc(0, "version", "The message bus version."); orb.addMethod(method); diff --git a/messagebus/src/tests/replygate/replygate.cpp b/messagebus/src/tests/replygate/replygate.cpp index 0acdc0a5611..c71993c9368 100644 --- a/messagebus/src/tests/replygate/replygate.cpp +++ b/messagebus/src/tests/replygate/replygate.cpp @@ -9,58 +9,59 @@ using namespace mbus; -struct MyGate : public ReplyGate -{ +namespace { + +struct MyGate : public ReplyGate { static int ctorCnt; static int dtorCnt; + MyGate(IMessageHandler &sender) : ReplyGate(sender) { ++ctorCnt; } - virtual ~MyGate() { + + ~MyGate() override { ++dtorCnt; } }; + int MyGate::ctorCnt = 0; int MyGate::dtorCnt = 0; -struct MyReply : public EmptyReply -{ +struct MyReply : public EmptyReply { static int ctorCnt; static int dtorCnt; + MyReply() : EmptyReply() { ++ctorCnt; } - virtual ~MyReply() { + + ~MyReply() override { ++dtorCnt; } }; + int MyReply::ctorCnt = 0; int MyReply::dtorCnt = 0; -struct MySender : public IMessageHandler -{ +struct MySender : public IMessageHandler { // giving a sync reply here is against the API contract, but it is // ok for testing. void handleMessage(Message::UP msg) override { - Reply::UP reply(new MyReply()); + auto reply = std::make_unique<MyReply>(); msg->swapState(*reply); IReplyHandler &handler = reply->getCallStack().pop(*reply); handler.handleReply(std::move(reply)); } }; +} -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("replygate_test"); +TEST("replygate_test") { { RoutableQueue q; MySender sender; - MyGate *gate = new MyGate(sender); + auto gate = vespalib::make_ref_counted<MyGate>(sender); { - Message::UP msg(new SimpleMessage("test")); + auto msg = std::make_unique<SimpleMessage>("test"); msg->pushHandler(q); gate->handleMessage(std::move(msg)); } @@ -69,7 +70,7 @@ Test::Main() EXPECT_TRUE(MyReply::dtorCnt == 0); gate->close(); { - Message::UP msg(new SimpleMessage("test")); + auto msg = std::make_unique<SimpleMessage>("test"); msg->pushHandler(q); gate->handleMessage(std::move(msg)); } @@ -78,11 +79,12 @@ Test::Main() EXPECT_TRUE(MyReply::dtorCnt == 1); EXPECT_TRUE(MyGate::ctorCnt == 1); EXPECT_TRUE(MyGate::dtorCnt == 0); - gate->subRef(); + gate = vespalib::ref_counted<MyGate>(); EXPECT_TRUE(MyGate::ctorCnt == 1); EXPECT_TRUE(MyGate::dtorCnt == 1); } EXPECT_TRUE(MyReply::ctorCnt == 2); EXPECT_TRUE(MyReply::dtorCnt == 2); - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/messagebus/src/tests/sequencer/sequencer.cpp b/messagebus/src/tests/sequencer/sequencer.cpp index 25d9934fe8d..7a3aaab9b1f 100644 --- a/messagebus/src/tests/sequencer/sequencer.cpp +++ b/messagebus/src/tests/sequencer/sequencer.cpp @@ -5,6 +5,7 @@ #include <vespa/messagebus/routablequeue.h> #include <vespa/messagebus/emptyreply.h> #include <vespa/vespalib/testkit/testapp.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP("sequencer_test"); diff --git a/messagebus/src/vespa/messagebus/callstack.h b/messagebus/src/vespa/messagebus/callstack.h index 937a9e939b6..8b2ab206c18 100644 --- a/messagebus/src/vespa/messagebus/callstack.h +++ b/messagebus/src/vespa/messagebus/callstack.h @@ -72,9 +72,12 @@ public: * @param ctx The context to store. * @param discardHandler The handler for discarded messages. **/ - void push(IReplyHandler &replyHandler, Context ctx, IDiscardHandler *discardHandler = nullptr) { + void push(IReplyHandler &replyHandler, Context ctx, IDiscardHandler *discardHandler) { _stack.emplace_back(&replyHandler, discardHandler, ctx); } + void push(IReplyHandler &replyHandler, Context ctx) { + _stack.emplace_back(&replyHandler, nullptr, ctx); + } /** * Pop a frame from this stack. The handler part of the frame will diff --git a/messagebus/src/vespa/messagebus/context.h b/messagebus/src/vespa/messagebus/context.h index edf875630e1..d6e7b4e82e5 100644 --- a/messagebus/src/vespa/messagebus/context.h +++ b/messagebus/src/vespa/messagebus/context.h @@ -17,11 +17,10 @@ struct Context { /** * This is a region of memory that can be interpreted as either an - * integer, a floating-point number or a pointer. + * integer or a pointer. **/ union { uint64_t UINT64; - double DOUBLE; void *PTR; } value; @@ -38,13 +37,6 @@ struct Context Context(uint64_t v) { value.UINT64 = v; } /** - * Create a context from a floating-point number - * - * @param v the value - **/ - Context(double v) { value.DOUBLE = v; } - - /** * Create a context from a pointer * * @param pt the pointer diff --git a/messagebus/src/vespa/messagebus/idiscardhandler.h b/messagebus/src/vespa/messagebus/idiscardhandler.h index 049b8dfb245..5618402f14d 100644 --- a/messagebus/src/vespa/messagebus/idiscardhandler.h +++ b/messagebus/src/vespa/messagebus/idiscardhandler.h @@ -15,7 +15,7 @@ public: /** * Virtual destructor required for inheritance. */ - virtual ~IDiscardHandler() { } + virtual ~IDiscardHandler() = default; /** * This method is invoked by message bus when a routable is being diff --git a/messagebus/src/vespa/messagebus/intermediatesession.cpp b/messagebus/src/vespa/messagebus/intermediatesession.cpp index 2b8830f07e8..61cd77c0165 100644 --- a/messagebus/src/vespa/messagebus/intermediatesession.cpp +++ b/messagebus/src/vespa/messagebus/intermediatesession.cpp @@ -4,6 +4,8 @@ #include "messagebus.h" #include "replygate.h" +using vespalib::make_ref_counted; + namespace mbus { IntermediateSession::IntermediateSession(MessageBus &mbus, const IntermediateSessionParams ¶ms) : @@ -11,14 +13,13 @@ IntermediateSession::IntermediateSession(MessageBus &mbus, const IntermediateSes _name(params.getName()), _msgHandler(params.getMessageHandler()), _replyHandler(params.getReplyHandler()), - _gate(new ReplyGate(_mbus)) + _gate(make_ref_counted<ReplyGate>(_mbus)) { } IntermediateSession::~IntermediateSession() { _gate->close(); close(); - _gate->subRef(); } void diff --git a/messagebus/src/vespa/messagebus/intermediatesession.h b/messagebus/src/vespa/messagebus/intermediatesession.h index 0a17aa1e42a..8d938b6cb9e 100644 --- a/messagebus/src/vespa/messagebus/intermediatesession.h +++ b/messagebus/src/vespa/messagebus/intermediatesession.h @@ -4,11 +4,11 @@ #include "reply.h" #include "imessagehandler.h" #include "intermediatesessionparams.h" +#include "replygate.h" namespace mbus { class MessageBus; -class ReplyGate; class Message; /** @@ -21,12 +21,13 @@ class IntermediateSession : public IMessageHandler, private: friend class MessageBus; using MessageUP = std::unique_ptr<Message>; + template <typename T> using ref_counted = vespalib::ref_counted<T>; - MessageBus &_mbus; - string _name; - IMessageHandler &_msgHandler; - IReplyHandler &_replyHandler; - ReplyGate *_gate; + MessageBus &_mbus; + string _name; + IMessageHandler &_msgHandler; + IReplyHandler &_replyHandler; + ref_counted<ReplyGate> _gate; /** * This constructor is declared package private since only MessageBus is supposed to instantiate it. diff --git a/messagebus/src/vespa/messagebus/messenger.cpp b/messagebus/src/vespa/messagebus/messenger.cpp index 056be51609f..926d8b6ef22 100644 --- a/messagebus/src/vespa/messagebus/messenger.cpp +++ b/messagebus/src/vespa/messagebus/messenger.cpp @@ -18,70 +18,6 @@ struct DeleteFunctor } }; -class MessageTask : public mbus::Messenger::ITask { -private: - mbus::Message::UP _msg; - mbus::IMessageHandler &_handler; - -public: - MessageTask(mbus::Message::UP msg, mbus::IMessageHandler &handler) - : _msg(std::move(msg)), - _handler(handler) - { - // empty - } - - ~MessageTask() { - if (_msg) { - _msg->discard(); - } - } - - void run() override { - _handler.handleMessage(std::move(_msg)); - } - - uint8_t priority() const override { - if (_msg) { - return _msg->priority(); - } - - return 255; - } -}; - -class ReplyTask : public mbus::Messenger::ITask { -private: - mbus::Reply::UP _reply; - mbus::IReplyHandler &_handler; - -public: - ReplyTask(mbus::Reply::UP reply, mbus::IReplyHandler &handler) - : _reply(std::move(reply)), - _handler(handler) - { - // empty - } - - ~ReplyTask() { - if (_reply) { - _reply->discard(); - } - } - - void run() override { - _handler.handleReply(std::move(_reply)); - } - - uint8_t priority() const override { - if (_reply) { - return _reply->priority(); - } - - return 255; - } -}; - class SyncTask : public mbus::Messenger::ITask { private: vespalib::Gate &_gate; @@ -89,17 +25,13 @@ private: public: SyncTask(vespalib::Gate &gate) : _gate(gate) - { - // empty - } + { } ~SyncTask() { _gate.countDown(); } - void run() override { - // empty - } + void run() override { } uint8_t priority() const override { return 255; @@ -115,9 +47,7 @@ public: AddRecurrentTask(std::vector<ITask*> &tasks, mbus::Messenger::ITask::UP task) : _tasks(tasks), _task(std::move(task)) - { - // empty - } + { } void run() override { _tasks.push_back(_task.release()); @@ -136,9 +66,7 @@ public: DiscardRecurrentTasks(vespalib::Gate &gate, std::vector<ITask*> &tasks) : SyncTask(gate), _tasks(tasks) - { - // empty - } + { } void run() override { std::for_each(_tasks.begin(), _tasks.end(), DeleteFunctor<ITask>()); @@ -157,10 +85,11 @@ namespace mbus { Messenger::Messenger() : _lock(), - _pool(), + _cond(), _children(), _queue(), - _closed(false) + _closed(false), + _thread() {} Messenger::~Messenger() @@ -170,8 +99,9 @@ Messenger::~Messenger() _closed = true; } _cond.notify_all(); - - _pool.Close(); + if (_thread.joinable()) { + _thread.join(); + } std::for_each(_children.begin(), _children.end(), DeleteFunctor<ITask>()); if ( ! _queue.empty()) { LOG(warning, @@ -185,10 +115,8 @@ Messenger::~Messenger() } void -Messenger::Run(FastOS_ThreadInterface *thread, void *arg) +Messenger::run() { - (void)thread; - (void)arg; while (true) { ITask::UP task; { @@ -235,9 +163,7 @@ Messenger::discardRecurrentTasks() bool Messenger::start() { - if (_pool.NewThread(this) == nullptr) { - return false; - } + _thread = std::thread([this](){run();}); return true; } diff --git a/messagebus/src/vespa/messagebus/messenger.h b/messagebus/src/vespa/messagebus/messenger.h index 6ec42ed9c4c..4e3632af414 100644 --- a/messagebus/src/vespa/messagebus/messenger.h +++ b/messagebus/src/vespa/messagebus/messenger.h @@ -7,7 +7,9 @@ #include "reply.h" #include <vespa/vespalib/util/executor.h> #include <vespa/vespalib/util/arrayqueue.hpp> -#include <vespa/fastos/thread.h> +#include <condition_variable> +#include <mutex> +#include <thread> namespace mbus { @@ -16,7 +18,7 @@ namespace mbus { * tasks. Tasks are enqueued using the synchronized {@link #enqueue(Task)} * method, and are run in the order they were enqueued. */ -class Messenger : public FastOS_Runnable { +class Messenger { public: /** * Defines the required interface for tasks to be posted to this worker. @@ -39,15 +41,15 @@ public: }; private: - mutable std::mutex _lock; - std::condition_variable _cond; - FastOS_ThreadPool _pool; - std::vector<ITask*> _children; - vespalib::ArrayQueue<ITask*> _queue; - bool _closed; - + mutable std::mutex _lock; + std::condition_variable _cond; + std::vector<ITask*> _children; + vespalib::ArrayQueue<ITask*> _queue; + bool _closed; + std::thread _thread; + protected: - void Run(FastOS_ThreadInterface *thread, void *arg) override; + void run(); public: Messenger(); @@ -55,7 +57,7 @@ public: /** * Frees any allocated resources. Also destroys all queued tasks. */ - ~Messenger() override; + ~Messenger(); /** * Adds a recurrent task to this that is to be run for every iteration of diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp index 030e3f956e1..8ab9aa13394 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp @@ -18,7 +18,6 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/fastos/thread.h> #include <thread> #include <vespa/log/log.h> @@ -95,7 +94,6 @@ RPCNetwork::SendContext::handleVersion(const vespalib::Version *version) RPCNetwork::RPCNetwork(const RPCNetworkParams ¶ms) : _owner(nullptr), _ident(params.getIdentity()), - _threadPool(std::make_unique<FastOS_ThreadPool>()), _transport(std::make_unique<FNET_Transport>(toFNETConfig(params))), _orb(std::make_unique<FRT_Supervisor>(_transport.get())), _scheduler(*_transport->GetScheduler()), @@ -196,7 +194,7 @@ RPCNetwork::getSendAdapter(const vespalib::Version &version) bool RPCNetwork::start() { - if (!_transport->Start(_threadPool.get())) { + if (!_transport->Start()) { return false; } if (!_orb->Listen(_requestedPort)) { @@ -391,7 +389,6 @@ RPCNetwork::shutdown() // Unschedule any pending target pool flush task that may race with shutdown target flushing _scheduler.Kill(_targetPoolTask.get()); _transport->ShutDown(true); - _threadPool->Close(); } void diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.h b/messagebus/src/vespa/messagebus/network/rpcnetwork.h index 0d2435e5dcd..8e296981458 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.h +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.h @@ -16,7 +16,6 @@ #include <vespa/fnet/frt/invokable.h> class FNET_Transport; -class FastOS_ThreadPool; namespace slobrok { namespace api { class RegisterAPI; } @@ -56,7 +55,6 @@ private: INetworkOwner *_owner; Identity _ident; - std::unique_ptr<FastOS_ThreadPool> _threadPool; std::unique_ptr<FNET_Transport> _transport; std::unique_ptr<FRT_Supervisor> _orb; FNET_Scheduler &_scheduler; diff --git a/messagebus/src/vespa/messagebus/network/rpcsend.cpp b/messagebus/src/vespa/messagebus/network/rpcsend.cpp index 8c67424d5f2..e12313af53c 100644 --- a/messagebus/src/vespa/messagebus/network/rpcsend.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcsend.cpp @@ -93,7 +93,7 @@ RPCSend::handleDiscard(Context ctx) ReplyContext::UP tmp(static_cast<ReplyContext*>(ctx.value.PTR)); FRT_RPCRequest &req = tmp->getRequest(); FNET_Channel *chn = req.GetContext()._value.CHANNEL; - req.SubRef(); + req.internal_subref(); chn->Free(); } @@ -189,7 +189,7 @@ RPCSend::doRequestDone(FRT_RPCRequest *req) { reply->addError(error); } _net->getOwner().deliverReply(std::move(reply), ctx->getRecipient()); - req->SubRef(); + req->internal_subref(); } std::unique_ptr<Reply> diff --git a/messagebus/src/vespa/messagebus/network/rpctarget.cpp b/messagebus/src/vespa/messagebus/network/rpctarget.cpp index 656ab081652..9c6ca9dff69 100644 --- a/messagebus/src/vespa/messagebus/network/rpctarget.cpp +++ b/messagebus/src/vespa/messagebus/network/rpctarget.cpp @@ -19,7 +19,7 @@ RPCTarget::RPCTarget(const string &spec, FRT_Supervisor &orb) : RPCTarget::~RPCTarget() { - _target.SubRef(); + _target.internal_subref(); } void @@ -94,7 +94,7 @@ RPCTarget::RequestDone(FRT_RPCRequest *req) _state = (_version.get() ? VERSION_RESOLVED : VERSION_NOT_RESOLVED); } _cond.notify_all(); - req->SubRef(); + req->internal_subref(); } } // namespace mbus diff --git a/messagebus/src/vespa/messagebus/replygate.cpp b/messagebus/src/vespa/messagebus/replygate.cpp index d1bd6ef05c7..1028d46aff4 100644 --- a/messagebus/src/vespa/messagebus/replygate.cpp +++ b/messagebus/src/vespa/messagebus/replygate.cpp @@ -6,7 +6,6 @@ namespace mbus { ReplyGate::ReplyGate(IMessageHandler &sender) : - vespalib::ReferenceCounter(), _sender(sender), _open(true) { } @@ -14,7 +13,7 @@ ReplyGate::ReplyGate(IMessageHandler &sender) : void ReplyGate::handleMessage(Message::UP msg) { - addRef(); + internal_addref(); msg->pushHandler(*this, *this); _sender.handleMessage(std::move(msg)); } @@ -34,14 +33,13 @@ ReplyGate::handleReply(Reply::UP reply) } else { reply->discard(); } - subRef(); + internal_subref(); } void -ReplyGate::handleDiscard(Context ctx) +ReplyGate::handleDiscard(Context) { - (void)ctx; - subRef(); + internal_subref(); } } // namespace mbus diff --git a/messagebus/src/vespa/messagebus/replygate.h b/messagebus/src/vespa/messagebus/replygate.h index 0c487de3ecf..e09dfe63365 100644 --- a/messagebus/src/vespa/messagebus/replygate.h +++ b/messagebus/src/vespa/messagebus/replygate.h @@ -5,7 +5,7 @@ #include "idiscardhandler.h" #include "imessagehandler.h" #include "ireplyhandler.h" -#include <vespa/vespalib/util/referencecounter.h> +#include <vespa/vespalib/util/ref_counted.h> #include <atomic> namespace mbus { @@ -20,7 +20,7 @@ namespace mbus { * is handled outside this class. Note that this class is only intended for * internal use. */ -class ReplyGate : public vespalib::ReferenceCounter, +class ReplyGate : public vespalib::enable_ref_counted, public IDiscardHandler, public IMessageHandler, public IReplyHandler diff --git a/messagebus/src/vespa/messagebus/sendproxy.cpp b/messagebus/src/vespa/messagebus/sendproxy.cpp index dcb24b70eeb..fa2994a6b7e 100644 --- a/messagebus/src/vespa/messagebus/sendproxy.cpp +++ b/messagebus/src/vespa/messagebus/sendproxy.cpp @@ -13,9 +13,7 @@ SendProxy::SendProxy(MessageBus &mbus, INetwork &net, Resender *resender) : _msg(), _logTrace(false), _root() -{ - // empty -} +{ } void SendProxy::handleMessage(Message::UP msg) @@ -31,14 +29,13 @@ SendProxy::handleMessage(Message::UP msg) } } _msg = std::move(msg); - _root.reset(new RoutingNode(_mbus, _net, _resender, *this, *_msg, this)); + _root = std::make_unique<RoutingNode>(_mbus, _net, _resender, *this, *_msg, this); _root->send(); } void -SendProxy::handleDiscard(Context ctx) +SendProxy::handleDiscard(Context) { - (void)ctx; _msg->discard(); delete this; } diff --git a/messagebus/src/vespa/messagebus/sequencer.cpp b/messagebus/src/vespa/messagebus/sequencer.cpp index e17509033d6..59e3164e11d 100644 --- a/messagebus/src/vespa/messagebus/sequencer.cpp +++ b/messagebus/src/vespa/messagebus/sequencer.cpp @@ -18,8 +18,8 @@ Sequencer::Sequencer(IMessageHandler &sender) : Sequencer::~Sequencer() { - for (QueueMap::iterator it = _seqMap.begin(); it != _seqMap.end(); ++it) { - MessageQueue *queue = it->second; + for (auto & entry : _seqMap) { + MessageQueue *queue = entry.second; if (queue != nullptr) { while (queue->size() > 0) { Message *msg = queue->front(); @@ -39,7 +39,7 @@ Sequencer::filter(Message::UP msg) msg->setContext(Context(seqId)); { std::lock_guard guard(_lock); - QueueMap::iterator it = _seqMap.find(seqId); + auto it = _seqMap.find(seqId); if (it != _seqMap.end()) { if (it->second == nullptr) { it->second = new MessageQueue(); @@ -48,7 +48,7 @@ Sequencer::filter(Message::UP msg) make_string("Sequencer queued message with sequence id '%" PRIu64 "'.", seqId)); it->second->push(msg.get()); msg.release(); - return Message::UP(); + return {}; } _seqMap[seqId] = nullptr; // insert empty queue } @@ -87,7 +87,7 @@ Sequencer::handleReply(Reply::UP reply) Message::UP msg; { std::lock_guard guard(_lock); - QueueMap::iterator it = _seqMap.find(seq); + auto it = _seqMap.find(seq); MessageQueue *que = it->second; assert(it != _seqMap.end()); if (que == nullptr || que->size() == 0) { diff --git a/messagebus/src/vespa/messagebus/sourcesession.cpp b/messagebus/src/vespa/messagebus/sourcesession.cpp index 0691e0c07f9..feff3beed58 100644 --- a/messagebus/src/vespa/messagebus/sourcesession.cpp +++ b/messagebus/src/vespa/messagebus/sourcesession.cpp @@ -6,15 +6,17 @@ #include "tracelevel.h" #include <vespa/messagebus/routing/routingtable.h> #include <vespa/vespalib/util/stringfmt.h> +#include <cassert> using vespalib::make_string; +using vespalib::make_ref_counted; namespace mbus { SourceSession::SourceSession(MessageBus &mbus, const SourceSessionParams ¶ms) : _lock(), _mbus(mbus), - _gate(new ReplyGate(_mbus)), + _gate(make_ref_counted<ReplyGate>(_mbus)), _sequencer(*_gate), _replyHandler(params.getReplyHandler()), _throttlePolicy(params.getThrottlePolicy()), @@ -31,9 +33,6 @@ SourceSession::~SourceSession() // Ensure that no more replies propagate from mbus. _gate->close(); _mbus.sync(); - - // Tell gate that we will no longer use it. - _gate->subRef(); } Result diff --git a/messagebus/src/vespa/messagebus/sourcesession.h b/messagebus/src/vespa/messagebus/sourcesession.h index d918724ad4f..03628f06c56 100644 --- a/messagebus/src/vespa/messagebus/sourcesession.h +++ b/messagebus/src/vespa/messagebus/sourcesession.h @@ -5,13 +5,13 @@ #include "result.h" #include "sequencer.h" #include "sourcesessionparams.h" +#include "replygate.h" #include <atomic> #include <condition_variable> namespace mbus { class MessageBus; -class ReplyGate; /** * A SourceSession is used to send Message objects along a named or explicitly defined route and get Reply @@ -21,11 +21,12 @@ class ReplyGate; class SourceSession : public IReplyHandler { private: friend class MessageBus; + template <typename T> using ref_counted = vespalib::ref_counted<T>; std::mutex _lock; std::condition_variable _cond; MessageBus &_mbus; - ReplyGate *_gate; + ref_counted<ReplyGate> _gate; Sequencer _sequencer; IReplyHandler &_replyHandler; IThrottlePolicy::SP _throttlePolicy; diff --git a/messagebus/src/vespa/messagebus/testlib/slobrok.cpp b/messagebus/src/vespa/messagebus/testlib/slobrok.cpp index 889daf538a3..b4cd2c846aa 100644 --- a/messagebus/src/vespa/messagebus/testlib/slobrok.cpp +++ b/messagebus/src/vespa/messagebus/testlib/slobrok.cpp @@ -9,80 +9,37 @@ #include <vespa/log/log.h> LOG_SETUP(".slobrok"); -namespace { -class WaitTask : public FNET_Task -{ -private: - bool _done; - std::mutex _mon; - std::condition_variable _cond; -public: - explicit WaitTask(FNET_Scheduler *s) : FNET_Task(s), _done(false), _mon() {} - ~WaitTask() override; - void wait() { - std::unique_lock guard(_mon); - while (!_done) { - _cond.wait(guard); - } - } - - void PerformTask() override { - std::lock_guard guard(_mon); - _done = true; - _cond.notify_one(); - } -}; - -WaitTask::~WaitTask() = default; -} // namespace <unnamed> - namespace mbus { void -Slobrok::Thread::setEnv(slobrok::SBEnv *env) -{ - _env = env; -} - -void -Slobrok::Thread::Run(FastOS_ThreadInterface *, void *) -{ - if (_env->MainLoop() != 0) { - LOG_ABORT("Slobrok main failed"); - } -} - -void Slobrok::init() { slobrok::ConfigShim shim(_port); _env = std::make_unique<slobrok::SBEnv>(shim); - _thread.setEnv(_env.get()); - WaitTask wt(_env->getTransport()->GetScheduler()); - wt.ScheduleNow(); - if (_pool.NewThread(&_thread, nullptr) == nullptr) { - LOG_ABORT("Could not spawn thread"); - } - wt.wait(); + _thread = std::thread([env = _env.get()]() + { + if (env->MainLoop() != 0) { + LOG_ABORT("Slobrok main failed"); + } + }); + _env->getTransport()->sync(); int p = _env->getSupervisor()->GetListenPort(); LOG_ASSERT(p != 0 && (p == _port || _port == 0)); _port = p; } Slobrok::Slobrok() - : _pool(), - _env(), - _port(0), - _thread() + : _env(), + _port(0), + _thread() { init(); } Slobrok::Slobrok(int p) - : _pool(), - _env(), - _port(p), - _thread() + : _env(), + _port(p), + _thread() { init(); } @@ -90,7 +47,7 @@ Slobrok::Slobrok(int p) Slobrok::~Slobrok() { _env->getTransport()->ShutDown(true); - _pool.Close(); + _thread.join(); } int diff --git a/messagebus/src/vespa/messagebus/testlib/slobrok.h b/messagebus/src/vespa/messagebus/testlib/slobrok.h index 7810222f07d..ff8e9cbb89a 100644 --- a/messagebus/src/vespa/messagebus/testlib/slobrok.h +++ b/messagebus/src/vespa/messagebus/testlib/slobrok.h @@ -4,7 +4,7 @@ #include <vespa/messagebus/common.h> #include <vespa/slobrok/cfg.h> -#include <vespa/fastos/thread.h> +#include <thread> namespace slobrok { class SBEnv; @@ -15,17 +15,9 @@ namespace mbus { class Slobrok { private: - class Thread : public FastOS_Runnable { - private: - slobrok::SBEnv *_env; - public: - void setEnv(slobrok::SBEnv *env); - void Run(FastOS_ThreadInterface *, void *) override; - }; - FastOS_ThreadPool _pool; std::unique_ptr<slobrok::SBEnv> _env; - int _port; - Thread _thread; + int _port; + std::thread _thread; Slobrok(const Slobrok &); Slobrok &operator=(const Slobrok &); @@ -42,4 +34,3 @@ public: }; } // namespace mbus - diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java index ab1c2e70735..7ce8dd12b30 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java @@ -79,7 +79,7 @@ public class MetricsManager { * @return metrics for all matching services */ public List<MetricsPacket> getMetrics(List<VespaService> services, Instant startTime) { - MetricsPacket.Builder [] builderArray = getMetricsBuildersAsArray(services, startTime, null); + MetricsPacket.Builder[] builderArray = getMetricsBuildersAsArray(services, startTime, null); List<MetricsPacket> metricsPackets = new ArrayList<>(builderArray.length); for (int i = 0; i < builderArray.length; i++) { metricsPackets.add(builderArray[i].build()); @@ -87,6 +87,7 @@ public class MetricsManager { } return metricsPackets; } + public List<MetricsPacket> getMetrics(List<VespaService> services, Instant startTime, ConsumerId consumerId) { MetricsPacket.Builder [] builderArray = getMetricsBuildersAsArray(services, startTime, consumerId); List<MetricsPacket> metricsPackets = new ArrayList<>(builderArray.length); @@ -97,7 +98,7 @@ public class MetricsManager { return metricsPackets; } - private MetricsPacket.Builder [] getMetricsBuildersAsArray(List<VespaService> services, Instant startTime, ConsumerId consumerId) { + private MetricsPacket.Builder[] getMetricsBuildersAsArray(List<VespaService> services, Instant startTime, ConsumerId consumerId) { List<MetricsPacket.Builder> builders = getMetricsAsBuilders(services, startTime, consumerId); return builders.toArray(new MetricsPacket.Builder[0]); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java index 821636336a8..21c7c78a224 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java @@ -25,14 +25,10 @@ public class Node { } public Node(String role, String hostname, int port, String path) { - Objects.requireNonNull(role, "Null role is not allowed"); - Objects.requireNonNull(hostname, "Null hostname is not allowed"); - Objects.requireNonNull(path, "Null path is not allowed"); - - this.role = role; - this.hostname = hostname; + this.role = Objects.requireNonNull(role, "Null role is not allowed"); + this.hostname = Objects.requireNonNull(hostname, "Null hostname is not allowed"); this.port = port; - this.path = path; + this.path = Objects.requireNonNull(path, "Null path is not allowed"); metricsUriBase = "http://" + hostname + ":" + port + path; } @@ -55,10 +51,10 @@ public class Node { public int hashCode() { return Objects.hash(role, hostname, port, path); } + @Override public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(role).append(":").append(metricsUriBase); - return sb.toString(); + return role + ":" + metricsUriBase; } + } diff --git a/metrics/CMakeLists.txt b/metrics/CMakeLists.txt index d75c17d4a00..5b95d8635d4 100644 --- a/metrics/CMakeLists.txt +++ b/metrics/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib config_cloudconfig diff --git a/metrics/src/tests/metricmanagertest.cpp b/metrics/src/tests/metricmanagertest.cpp index 98d03514de0..9e6b0f40be3 100644 --- a/metrics/src/tests/metricmanagertest.cpp +++ b/metrics/src/tests/metricmanagertest.cpp @@ -5,12 +5,10 @@ #include <vespa/metrics/metricmanager.h> #include <vespa/metrics/state_api_adapter.h> #include <vespa/metrics/textwriter.h> -#include <vespa/metrics/xmlwriter.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/vespalib/util/xmlstream.h> #include <vespa/vespalib/util/time.h> #include <vespa/vespalib/data/simple_buffer.h> #include <vespa/vespalib/util/atomic.h> @@ -31,7 +29,7 @@ struct MetricManagerTest : public ::testing::Test { // MetricManager that aren't accessible to "freestanding" fixtures. So we // get the test to do the necessary poking and prodding for us instead. void takeSnapshots(MetricManager& mm, time_t timeToProcess) { - mm.takeSnapshots(mm.getMetricLock(), timeToProcess); + mm.takeSnapshots(mm.getMetricLock(), system_time(vespalib::from_s<system_time::duration>(timeToProcess))); } }; @@ -57,7 +55,7 @@ SubMetricSet::SubMetricSet(const Metric::String & name, MetricSet* owner) valsum.addMetricToSum(val1); valsum.addMetricToSum(val2); } -SubMetricSet::~SubMetricSet() { } +SubMetricSet::~SubMetricSet() = default; struct MultiSubMetricSet { @@ -167,20 +165,15 @@ getMatchedMetrics(const vespalib::string& config) MetricLockGuard g(mm.getMetricLock()); mm.visit(g, mm.getActiveMetrics(g), visitor, "consumer"); - MetricManager::ConsumerSpec::SP consumerSpec( - mm.getConsumerSpec(g, "consumer")); - return std::pair<std::string, std::string>( - visitor.toString(), - consumerSpec.get() ? consumerSpec->toString() - : "Non-existing consumer"); + const MetricManager::ConsumerSpec * consumerSpec = mm.getConsumerSpec(g, "consumer"); + return { visitor.toString(), consumerSpec ? consumerSpec->toString() : "Non-existing consumer" }; } } #define ASSERT_CONSUMER_MATCH(name, expected, config) \ { \ - std::pair<std::string, std::string> consumerMatch( \ - getMatchedMetrics(config)); \ + std::pair<std::string, std::string> consumerMatch(getMatchedMetrics(config)); \ EXPECT_EQ("\n" + expected, "\n" + consumerMatch.first) << (name + std::string(": ") + consumerMatch.second); \ } @@ -371,10 +364,10 @@ class FakeTimer : public MetricManager::Timer { std::atomic<time_t> _time; public: FakeTimer(time_t startTime = 0) : _time(startTime) {} - time_t getTime() const override { return load_relaxed(_time); } + time_point getTime() const override { return time_point(vespalib::from_s<time_point::duration>(load_relaxed(_time))); } void set_time(time_t t) noexcept { store_relaxed(_time, t); } // Not safe for multiple writers, only expected to be called by test. - void add_time(time_t t) noexcept { set_time(getTime() + t); } + void add_time(time_t t) noexcept { set_time(load_relaxed(_time) + t); } }; struct BriefValuePrinter : public MetricVisitor { @@ -391,55 +384,46 @@ struct BriefValuePrinter : public MetricVisitor { } }; -bool waitForTimeProcessed(const MetricManager& mm, - time_t processtime, uint32_t timeout = 120) +bool waitForTimeProcessed(const MetricManager& mm, time_point::duration processtime, uint32_t timeout = 120) { uint32_t lastchance = time(0) + timeout; while (time(0) < lastchance) { - if (mm.getLastProcessedTime() >= processtime) return true; + if (mm.getLastProcessedTime() >= time_point(processtime)) return true; mm.timeChangedNotification(); std::this_thread::sleep_for(10ms); } return false; } -std::string dumpAllSnapshots(const MetricManager& mm, - const std::string& consumer) +std::string dumpAllSnapshots(const MetricManager& mm, const std::string& consumer) { std::ostringstream ost; ost << "\n"; { MetricLockGuard metricLock(mm.getMetricLock()); BriefValuePrinter briefValuePrinter; - mm.visit(metricLock, mm.getActiveMetrics(metricLock), - briefValuePrinter, consumer); + mm.visit(metricLock, mm.getActiveMetrics(metricLock), briefValuePrinter, consumer); ost << "Current: " << briefValuePrinter.ost.str() << "\n"; } { MetricLockGuard metricLock(mm.getMetricLock()); BriefValuePrinter briefValuePrinter; - mm.visit(metricLock, mm.getTotalMetricSnapshot(metricLock), - briefValuePrinter, consumer); + mm.visit(metricLock, mm.getTotalMetricSnapshot(metricLock), briefValuePrinter, consumer); ost << "Total: " << briefValuePrinter.ost.str() << "\n"; } - std::vector<uint32_t> periods; - { - MetricLockGuard metricLock(mm.getMetricLock()); - periods = mm.getSnapshotPeriods(metricLock); - } - for (uint32_t i=0; i<periods.size(); ++i) { - MetricLockGuard metricLock(mm.getMetricLock()); - const MetricSnapshotSet& set(mm.getMetricSnapshotSet( - metricLock, periods[i])); + + MetricLockGuard metricLock(mm.getMetricLock()); + auto periods = mm.getSnapshotPeriods(metricLock); + for (vespalib::duration period : periods) { + const MetricSnapshotSet& set(mm.getMetricSnapshotSet(metricLock, period)); ost << set.getName() << "\n"; - uint32_t count = 0; - for (uint32_t j=0; j<2; ++j) { + for (uint32_t count=0,j=0; j<2; ++j) { if (set.getCount() == 1 && j == 1) continue; const MetricSnapshot& snap(set.getSnapshot(j == 1)); BriefValuePrinter briefValuePrinter; mm.visit(metricLock, snap, briefValuePrinter, consumer); ost << " " << count++ << " " << &snap.getMetrics() << ": " - << briefValuePrinter.ost.str() << "\n"; + << briefValuePrinter.ost.str() << "\n"; } } return ost.str(); @@ -451,31 +435,28 @@ std::string dumpAllSnapshots(const MetricManager& mm, { \ MetricLockGuard lockGuard(mm.getMetricLock()); \ BriefValuePrinter briefValuePrinter; \ - if (period == -1) { \ - mm.visit(lockGuard, mm.getActiveMetrics(lockGuard), \ - briefValuePrinter, "snapper"); \ - } else if (period == 0) { \ - mm.visit(lockGuard, mm.getTotalMetricSnapshot(lockGuard), \ - briefValuePrinter, "snapper"); \ + if (period < vespalib::duration::zero()) { \ + mm.visit(lockGuard, mm.getActiveMetrics(lockGuard), briefValuePrinter, "snapper"); \ + } else if (period == vespalib::duration::zero()) { \ + mm.visit(lockGuard, mm.getTotalMetricSnapshot(lockGuard), briefValuePrinter, "snapper"); \ } else { \ - mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, period), \ - briefValuePrinter, "snapper"); \ + mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, period), briefValuePrinter, "snapper"); \ } \ EXPECT_EQ(std::string(expected), briefValuePrinter.ost.str()) << dumpAllSnapshots(mm, "snapper"); \ } #define ASSERT_PROCESS_TIME(mm, time) \ { \ - LOG(info, "Waiting for processed time %u.", time); \ - bool gotToCorrectProgress = waitForTimeProcessed(mm, time); \ + LOG(info, "Waiting for processed time %s.", vespalib::to_string(time_point(time)).c_str()); \ + bool gotToCorrectProgress = waitForTimeProcessed(mm, (time)); \ if (!gotToCorrectProgress) \ FAIL() << "Failed to get to processed time within timeout"; \ } TEST_F(MetricManagerTest, test_snapshots) { - FakeTimer* timer = new FakeTimer(1000); - std::unique_ptr<MetricManager::Timer> timerImpl(timer); + auto timerImpl = std::make_unique<FakeTimer>(1000); + FakeTimer & timer = *timerImpl; TestMetricSet mySet; MetricManager mm(std::move(timerImpl)); { @@ -494,8 +475,7 @@ TEST_F(MetricManagerTest, test_snapshots) { MetricLockGuard lockGuard(mm.getMetricLock()); mm.visit(lockGuard, mm.getActiveMetrics(lockGuard), visitor, "snapper"); - MetricManager::ConsumerSpec::SP consumerSpec( - mm.getConsumerSpec(lockGuard, "snapper")); + const MetricManager::ConsumerSpec * consumerSpec = mm.getConsumerSpec(lockGuard, "snapper"); EXPECT_EQ(std::string("\n" "temp.val6\n" "temp.sub.val1\n" @@ -508,12 +488,11 @@ TEST_F(MetricManagerTest, test_snapshots) "*temp.multisub.sum.val1\n" "*temp.multisub.sum.val2\n" "*temp.multisub.sum.valsum\n"), - "\n" + visitor.toString()) << (consumerSpec.get() ? consumerSpec->toString() - : "Non-existing consumer"); + "\n" + visitor.toString()) << (consumerSpec ? consumerSpec->toString() : "Non-existing consumer"); } // Initially, there should be no metrics logged - ASSERT_PROCESS_TIME(mm, 1000); - ASSERT_VALUES(mm, 5 * 60, ""); + ASSERT_PROCESS_TIME(mm, 1000s); + ASSERT_VALUES(mm, 5 * 60s, ""); // Adding metrics done in first five minutes. mySet.val6.addValue(2); @@ -522,11 +501,11 @@ TEST_F(MetricManagerTest, test_snapshots) mySet.val10.a.val1.addValue(7); mySet.val10.a.val2.addValue(2); mySet.val10.b.val1.addValue(1); - timer->add_time(5 * 60); - ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60); - ASSERT_VALUES(mm, 5 * 60, "2,4,4,1,7,9,1,1,8,2,10"); - ASSERT_VALUES(mm, 60 * 60, ""); - ASSERT_VALUES(mm, 0 * 60, "2,4,4,1,7,9,1,1,8,2,10"); + timer.add_time(5 * 60); + ASSERT_PROCESS_TIME(mm, 1000s + 5 * 60s); + ASSERT_VALUES(mm, 5 * 60s, "2,4,4,1,7,9,1,1,8,2,10"); + ASSERT_VALUES(mm, 60 * 60s, ""); + ASSERT_VALUES(mm, 0 * 60s, "2,4,4,1,7,9,1,1,8,2,10"); // Adding metrics done in second five minute period. Total should // be updated to account for both @@ -536,120 +515,44 @@ TEST_F(MetricManagerTest, test_snapshots) mySet.val10.a.val1.addValue(8); mySet.val10.a.val2.addValue(3); mySet.val10.b.val1.addValue(2); - timer->add_time(5 * 60); - ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * 2); - ASSERT_VALUES(mm, 5 * 60, "4,5,5,1,8,11,2,2,10,3,13"); - ASSERT_VALUES(mm, 60 * 60, ""); - ASSERT_VALUES(mm, 0 * 60, "4,5,5,2,8,11,2,2,10,3,13"); + timer.add_time(5 * 60); + ASSERT_PROCESS_TIME(mm, 1000s + 5 * 60 * 2s); + ASSERT_VALUES(mm, 5 * 60s, "4,5,5,1,8,11,2,2,10,3,13"); + ASSERT_VALUES(mm, 60 * 60s, ""); + ASSERT_VALUES(mm, 0 * 60s, "4,5,5,2,8,11,2,2,10,3,13"); // Adding another five minute period where nothing have happened. // Metric for last 5 minutes should be 0. - timer->add_time(5 * 60); - ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * 3); - ASSERT_VALUES(mm, 5 * 60, "0,0,0,0,0,0,0,0,0,0,0"); - ASSERT_VALUES(mm, 60 * 60, ""); - ASSERT_VALUES(mm, 0 * 60, "4,5,5,2,8,11,2,2,10,3,13"); + timer.add_time(5 * 60); + ASSERT_PROCESS_TIME(mm, 1000s + 5 * 60s * 3); + ASSERT_VALUES(mm, 5 * 60s, "0,0,0,0,0,0,0,0,0,0,0"); + ASSERT_VALUES(mm, 60 * 60s, ""); + ASSERT_VALUES(mm, 0 * 60s, "4,5,5,2,8,11,2,2,10,3,13"); // Advancing time to 60 minute period, we should create a proper // 60 minute period timer. mySet.val6.addValue(6); for (uint32_t i=0; i<9; ++i) { // 9 x 5 minutes. Avoid snapshot bumping // due to taking snapshots in the past - timer->add_time(5 * 60); - ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * (4 + i)); + timer.add_time(5 * 60); + ASSERT_PROCESS_TIME(mm, 1000s + 5 * 60s * (4 + i)); } - ASSERT_VALUES(mm, 5 * 60, "0,0,0,0,0,0,0,0,0,0,0"); - ASSERT_VALUES(mm, 60 * 60, "6,5,5,2,8,11,2,2,10,3,13"); - ASSERT_VALUES(mm, 0 * 60, "6,5,5,2,8,11,2,2,10,3,13"); + ASSERT_VALUES(mm, 5 * 60s, "0,0,0,0,0,0,0,0,0,0,0"); + ASSERT_VALUES(mm, 60 * 60s, "6,5,5,2,8,11,2,2,10,3,13"); + ASSERT_VALUES(mm, 0 * 60s, "6,5,5,2,8,11,2,2,10,3,13"); // Test that reset works - mm.reset(1000); - ASSERT_VALUES(mm, -1, "0,0,0,0,0,0,0,0,0,0,0"); - ASSERT_VALUES(mm, 5 * 60, "0,0,0,0,0,0,0,0,0,0,0"); - ASSERT_VALUES(mm, 60 * 60, "0,0,0,0,0,0,0,0,0,0,0"); - ASSERT_VALUES(mm, 0 * 60, "0,0,0,0,0,0,0,0,0,0,0"); -} - -TEST_F(MetricManagerTest, test_xml_output) -{ - FakeTimer* timer = new FakeTimer(1000); - std::unique_ptr<MetricManager::Timer> timerImpl(timer); - MetricManager mm(std::move(timerImpl)); - TestMetricSet mySet; - { - MetricLockGuard lockGuard(mm.getMetricLock()); - mm.registerMetric(lockGuard, mySet.set); - } - - // Initialize metric manager to get snapshots created. - mm.init(ConfigUri("raw:" - "consumer[2]\n" - "consumer[0].name snapper\n" - "consumer[0].tags[1]\n" - "consumer[0].tags[0] snaptest\n" - "consumer[1].name log\n" - "consumer[1].tags[1]\n" - "consumer[1].tags[0] snaptest\n")); - - takeSnapshots(mm, 1000); - - // Adding metrics to have some values in them - mySet.val6.addValue(2); - mySet.val9.val1.addValue(4); - mySet.val10.count.inc(); - mySet.val10.a.val1.addValue(7); - mySet.val10.a.val2.addValue(2); - mySet.val10.b.val1.addValue(1); - - timer->set_time(1300); - takeSnapshots(mm, 1300); - - std::string expected( - "'<snapshot name=\"5 minute\" from=\"1000\" to=\"1300\" period=\"300\">\n" - " <temp>\n" - " <val6 average=\"2\" last=\"2\" min=\"2\" max=\"2\" count=\"1\"/>\n" - " <sub>\n" - " <val1 average=\"4\" last=\"4\" min=\"4\" max=\"4\" count=\"1\"/>\n" - " <valsum average=\"4\" last=\"4\" min=\"4\" max=\"4\" count=\"1\"/>\n" - " </sub>\n" - " <multisub>\n" - " <count count=\"1\"/>\n" - " <a>\n" - " <val1 average=\"7\" last=\"7\" min=\"7\" max=\"7\" count=\"1\"/>\n" - " <valsum average=\"9\" last=\"9\"/>\n" - " </a>\n" - " <b>\n" - " <val1 average=\"1\" last=\"1\" min=\"1\" max=\"1\" count=\"1\"/>\n" - " <valsum average=\"1\" last=\"1\" min=\"1\" max=\"1\" count=\"1\"/>\n" - " </b>\n" - " <sum>\n" - " <val1 average=\"8\" last=\"8\"/>\n" - " <val2 average=\"2\" last=\"2\" min=\"2\" max=\"2\" count=\"1\"/>\n" - " <valsum average=\"10\" last=\"10\"/>\n" - " </sum>\n" - " </multisub>\n" - " </temp>\n" - "</snapshot>'"); - - std::ostringstream ost; - vespalib::XmlOutputStream xos(ost, " "); - XmlWriter writer(xos, 300, 0); - { - MetricLockGuard lockGuard(mm.getMetricLock()); - mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, 300, false), - writer, "snapper"); - } - std::string actual(ost.str()); - // Not bothering to match all the nitty gritty details as it will test - // more than it needs to. Just be here in order to check on XML output - // easily if needed. - EXPECT_EQ(expected, "'" + actual + "'"); + mm.reset(system_time(1000s)); + ASSERT_VALUES(mm, -1s, "0,0,0,0,0,0,0,0,0,0,0"); + ASSERT_VALUES(mm, 5 * 60s, "0,0,0,0,0,0,0,0,0,0,0"); + ASSERT_VALUES(mm, 60 * 60s, "0,0,0,0,0,0,0,0,0,0,0"); + ASSERT_VALUES(mm, 0 * 60s, "0,0,0,0,0,0,0,0,0,0,0"); } TEST_F(MetricManagerTest, test_json_output) { - FakeTimer* timer = new FakeTimer(1000); - std::unique_ptr<MetricManager::Timer> timerImpl(timer); + auto timerImpl = std::make_unique<FakeTimer>(1000); + FakeTimer & timer = *timerImpl; MetricManager mm(std::move(timerImpl)); TestMetricSet mySet; { @@ -674,7 +577,7 @@ TEST_F(MetricManagerTest, test_json_output) mySet.val10.a.val2.addValue(2); mySet.val10.b.val1.addValue(1); - timer->set_time(1300); + timer.set_time(1300); takeSnapshots(mm, 1300); // Create json output @@ -683,8 +586,7 @@ TEST_F(MetricManagerTest, test_json_output) JsonWriter writer(jsonStream); { MetricLockGuard lockGuard(mm.getMetricLock()); - mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, 300, false), - writer, "snapper"); + mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, 300s, false), writer, "snapper"); } jsonStream.finalize(); std::string jsonData = as.str(); @@ -773,22 +675,19 @@ struct MetricSnapshotTestFixture JsonWriter writer(jsonStream); { MetricLockGuard lockGuard(manager.getMetricLock()); - manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300, false), - writer, "snapper"); + manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300s, false), writer, "snapper"); } jsonStream.finalize(); return as.str(); } - std::string renderLastSnapshotAsText( - const std::string& matchPattern = ".*") const + std::string renderLastSnapshotAsText(const std::string& matchPattern = ".*") const { std::ostringstream ss; - TextWriter writer(ss, 300, matchPattern, true); + TextWriter writer(ss, 300s, matchPattern, true); { MetricLockGuard lockGuard(manager.getMetricLock()); - manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300, false), - writer, "snapper"); + manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300s, false), writer, "snapper"); } return ss.str(); } @@ -819,8 +718,7 @@ public: } std::string nthMetricDimension(size_t metricIndex, const std::string& key) { - return nthMetric(metricIndex)["dimensions"][key] - .asString().make_string(); + return nthMetric(metricIndex)["dimensions"][key].asString().make_string(); } // Verify that the nth metric has the given name and the given set of @@ -843,7 +741,7 @@ JsonMetricWrapper::JsonMetricWrapper(const std::string& jsonText) { vespalib::slime::JsonFormat::decode(vespalib::Memory(jsonText), _tree); } -JsonMetricWrapper::~JsonMetricWrapper() { } +JsonMetricWrapper::~JsonMetricWrapper() = default; struct DimensionTestMetricSet : MetricSet { @@ -851,7 +749,7 @@ struct DimensionTestMetricSet : MetricSet LongCountMetric val2; DimensionTestMetricSet(MetricSet* owner = nullptr); - ~DimensionTestMetricSet(); + ~DimensionTestMetricSet() override; }; DimensionTestMetricSet::DimensionTestMetricSet(MetricSet* owner) @@ -859,7 +757,7 @@ DimensionTestMetricSet::DimensionTestMetricSet(MetricSet* owner) val1("val1", {{"tag1"}}, "val1 desc", this), val2("val2", {{"baz", "superbaz"}}, "val2 desc", this) { } -DimensionTestMetricSet::~DimensionTestMetricSet() { } +DimensionTestMetricSet::~DimensionTestMetricSet() = default; } @@ -976,9 +874,7 @@ TEST_F(MetricManagerTest, json_output_can_have_multiple_sets_with_same_name) TEST_F(MetricManagerTest, test_text_output) { - FakeTimer* timer = new FakeTimer(1000); - std::unique_ptr<MetricManager::Timer> timerImpl(timer); - MetricManager mm(std::move(timerImpl)); + MetricManager mm(std::make_unique<FakeTimer>(1000)); TestMetricSet mySet; { MetricLockGuard lockGuard(mm.getMetricLock()); @@ -1001,7 +897,7 @@ TEST_F(MetricManagerTest, test_text_output) "consumer[1].tags[1]\n" "consumer[1].tags[0] snaptest\n")); std::string expected( - "snapshot \"Active metrics showing updates since last snapshot\" from 1000 to 0 period 0\n" + "snapshot \"Active metrics showing updates since last snapshot\" from 1970-01-01 00:16:40.000 UTC to 1970-01-01 00:00:00.000 UTC period 0\n" "temp.val6 average=2 last=2 min=2 max=2 count=1 total=2\n" "temp.sub.val1 average=4 last=4 min=4 max=4 count=1 total=4\n" "temp.sub.valsum average=4 last=4 min=4 max=4 count=1 total=4\n" @@ -1014,7 +910,7 @@ TEST_F(MetricManagerTest, test_text_output) "temp.multisub.sum.val2 average=2 last=2 min=2 max=2 count=1 total=2\n" "temp.multisub.sum.valsum average=10 last=10"); std::ostringstream ost; - TextWriter writer(ost, 300, ".*", true); + TextWriter writer(ost, 300s, ".*", true); { MetricLockGuard lockGuard(mm.getMetricLock()); mm.visit(lockGuard, mm.getActiveMetrics(lockGuard), writer, "snapper"); @@ -1037,11 +933,9 @@ TEST_F(MetricManagerTest, text_output_supports_dimensions) fixture.takeSnapshotsOnce(); std::string actual = fixture.renderLastSnapshotAsText("outer.*temp.*val"); std::string expected( - "snapshot \"5 minute\" from 1000 to 1300 period 300\n" - "outer{fancy:stuff}.temp{bar:hyperbar,foo:megafoo}.val1 " - "average=2 last=2 min=2 max=2 count=1 total=2\n" - "outer{fancy:stuff}.temp{bar:hyperbar,foo:megafoo}.val2" - "{baz:superbaz} count=1"); + "snapshot \"5 minute\" from 1970-01-01 00:16:40.000 UTC to 1970-01-01 00:21:40.000 UTC period 300\n" + "outer{fancy:stuff}.temp{bar:hyperbar,foo:megafoo}.val1 average=2 last=2 min=2 max=2 count=1 total=2\n" + "outer{fancy:stuff}.temp{bar:hyperbar,foo:megafoo}.val2{baz:superbaz} count=1"); EXPECT_EQ(expected, actual); } @@ -1051,11 +945,8 @@ namespace { std::mutex& _output_mutex; FakeTimer& _timer; - MyUpdateHook(std::ostringstream& output, - std::mutex& output_mutex, - const char* name, - FakeTimer& timer) - : UpdateHook(name), + MyUpdateHook(std::ostringstream& output, std::mutex& output_mutex, const char* name, vespalib::system_clock::duration period, FakeTimer& timer) + : UpdateHook(name, period), _output(output), _output_mutex(output_mutex), _timer(timer) @@ -1064,7 +955,7 @@ namespace { void updateMetrics(const MetricLockGuard & ) override { std::lock_guard lock(_output_mutex); // updateMetrics() called from metric manager thread - _output << _timer.getTime() << ": " << getName() << " called\n"; + _output << vespalib::count_s(_timer.getTime().time_since_epoch()) << ": " << getName() << " called\n"; } }; } @@ -1073,8 +964,8 @@ TEST_F(MetricManagerTest, test_update_hooks) { std::mutex output_mutex; std::ostringstream output; - FakeTimer* timer = new FakeTimer(1000); - std::unique_ptr<MetricManager::Timer> timerImpl(timer); + auto timerImpl = std::make_unique<FakeTimer>(1000); + FakeTimer & timer = *timerImpl; // Add a metric set just so one exist TestMetricSet mySet; MetricManager mm(std::move(timerImpl)); @@ -1083,12 +974,12 @@ TEST_F(MetricManagerTest, test_update_hooks) mm.registerMetric(lockGuard, mySet.set); } - MyUpdateHook preInitShort(output, output_mutex, "BIS", *timer); - MyUpdateHook preInitLong(output, output_mutex, "BIL", *timer); - MyUpdateHook preInitInfinite(output, output_mutex, "BII", *timer); - mm.addMetricUpdateHook(preInitShort, 5); - mm.addMetricUpdateHook(preInitLong, 300); - mm.addMetricUpdateHook(preInitInfinite, 0); + MyUpdateHook preInitShort(output, output_mutex, "BIS", 5s, timer); + MyUpdateHook preInitLong(output, output_mutex, "BIL", 300s, timer); + MyUpdateHook preInitInfinite(output, output_mutex, "BII", 0s, timer); + mm.addMetricUpdateHook(preInitShort); + mm.addMetricUpdateHook(preInitLong); + mm.addMetricUpdateHook(preInitInfinite); // All hooks should get called during initialization @@ -1104,56 +995,56 @@ TEST_F(MetricManagerTest, test_update_hooks) "consumer[1].tags[0] snaptest\n")); output << "Init done\n"; - MyUpdateHook postInitShort(output, output_mutex, "AIS", *timer); - MyUpdateHook postInitLong(output, output_mutex, "AIL", *timer); - MyUpdateHook postInitInfinite(output, output_mutex, "AII", *timer); - mm.addMetricUpdateHook(postInitShort, 5); - mm.addMetricUpdateHook(postInitLong, 400); - mm.addMetricUpdateHook(postInitInfinite, 0); + MyUpdateHook postInitShort(output, output_mutex, "AIS", 5s, timer); + MyUpdateHook postInitLong(output, output_mutex, "AIL", 400s, timer); + MyUpdateHook postInitInfinite(output, output_mutex, "AII", 0s, timer); + mm.addMetricUpdateHook(postInitShort); + mm.addMetricUpdateHook(postInitLong); + mm.addMetricUpdateHook(postInitInfinite); // After 5 seconds the short ones should get another. - timer->set_time(1006); - waitForTimeProcessed(mm, 1006); + timer.set_time(1006); + waitForTimeProcessed(mm, 1006s); // After 4 more seconds the short ones should get another // since last update was a second late. (Stable periods, process time // should not affect how often they are updated) - timer->set_time(1010); - waitForTimeProcessed(mm, 1010); + timer.set_time(1010); + waitForTimeProcessed(mm, 1010s); // Bumping considerably ahead, such that next update is in the past, // we should only get one update called in this period. - timer->set_time(1200); - waitForTimeProcessed(mm, 1200); + timer.set_time(1200); + waitForTimeProcessed(mm, 1200s); // No updates at this time. - timer->set_time(1204); - waitForTimeProcessed(mm, 1204); + timer.set_time(1204); + waitForTimeProcessed(mm, 1204s); // Give all hooks an update - mm.updateMetrics(true); + mm.updateMetrics(); // Last update should not have interfered with periods - timer->set_time(1205); - waitForTimeProcessed(mm, 1205); + timer.set_time(1205); + waitForTimeProcessed(mm, 1205s); // Time is just ahead of a snapshot. - timer->set_time(1299); - waitForTimeProcessed(mm, 1299); + timer.set_time(1299); + waitForTimeProcessed(mm, 1299s); // At time 1300 we are at a 5 minute snapshot bump // All hooks should thus get an update. The one with matching period // should only get one - timer->set_time(1300); - waitForTimeProcessed(mm, 1300); + timer.set_time(1300); + waitForTimeProcessed(mm, 1300s); // The snapshot time currently doesn't count for the metric at period // 400. It will get an event at this time. - timer->set_time(1450); - waitForTimeProcessed(mm, 1450); + timer.set_time(1450); + waitForTimeProcessed(mm, 1450s); std::string expected( "Running init\n" diff --git a/metrics/src/tests/snapshottest.cpp b/metrics/src/tests/snapshottest.cpp index b4eb4a1353c..580769bbadb 100644 --- a/metrics/src/tests/snapshottest.cpp +++ b/metrics/src/tests/snapshottest.cpp @@ -4,7 +4,6 @@ #include <vespa/metrics/metrics.h> #include <vespa/metrics/summetric.hpp> #include <vespa/vespalib/gtest/gtest.h> -#include <vespa/vespalib/util/size_literals.h> namespace metrics { @@ -122,15 +121,15 @@ struct TestMetricSet : public MetricSet { SubMetricSet set2; SumMetric<SubMetricSet> setSum; - TestMetricSet(vespalib::stringref name, MetricSet* owner = 0); + TestMetricSet(vespalib::stringref name); ~TestMetricSet(); void incValues(); }; -TestMetricSet::TestMetricSet(vespalib::stringref name, MetricSet* owner) - : MetricSet(name, {}, "", owner), +TestMetricSet::TestMetricSet(vespalib::stringref name) + : MetricSet(name, {}, "", nullptr), set1("set1", this), set2("set2", this), setSum("setSum", {}, "", this) @@ -149,7 +148,7 @@ TestMetricSet::incValues() { struct FakeTimer : public MetricManager::Timer { uint32_t _timeInSecs; FakeTimer() : _timeInSecs(1) {} - time_t getTime() const override { return _timeInSecs; } + time_point getTime() const override { return time_point(vespalib::from_s<time_point::duration>(_timeInSecs)); } }; void ASSERT_VALUE(int32_t value, const MetricSnapshot & snapshot, const char *name) __attribute__((noinline)); @@ -166,8 +165,8 @@ void ASSERT_VALUE(int32_t value, const MetricSnapshot & snapshot, const char *na } struct SnapshotTest : public ::testing::Test { - time_t tick(MetricManager& mgr, time_t currentTime) { - return mgr.tick(mgr.getMetricLock(), currentTime); + void tick(MetricManager& mgr, time_t currentTime) { + mgr.tick(mgr.getMetricLock(), time_point(vespalib::from_s<time_point::duration>(currentTime))); } }; @@ -176,8 +175,7 @@ TEST_F(SnapshotTest, test_snapshot_two_days) TestMetricSet set("test"); FakeTimer* timer; - MetricManager mm( - std::unique_ptr<MetricManager::Timer>(timer = new FakeTimer)); + MetricManager mm(std::unique_ptr<MetricManager::Timer>(timer = new FakeTimer)); { MetricLockGuard lockGuard(mm.getMetricLock()); mm.registerMetric(lockGuard, set); @@ -215,15 +213,14 @@ TEST_F(SnapshotTest, test_snapshot_two_days) << "\n"; */ - const MetricSnapshot* snap = 0; // active snapshot MetricLockGuard lockGuard(mm.getMetricLock()); - snap = &mm.getActiveMetrics(lockGuard); + const MetricSnapshot* snap = &mm.getActiveMetrics(lockGuard); ASSERT_VALUE(0, *snap, "test.set1.set1.count1"); ASSERT_VALUE(0, *snap, "test.set1.set1.countSum"); // 5 minute snapshot - snap = &mm.getMetricSnapshot(lockGuard, 5 * 60); + snap = &mm.getMetricSnapshot(lockGuard, 5 * 60s); ASSERT_VALUE(1, *snap, "test.set1.set1.count1"); ASSERT_VALUE(2, *snap, "test.set1.set1.countSum"); @@ -231,7 +228,7 @@ TEST_F(SnapshotTest, test_snapshot_two_days) ASSERT_VALUE(1, *snap, "test.set1.set1.averageSum"); // 1 hour snapshot - snap = &mm.getMetricSnapshot(lockGuard, 60 * 60); + snap = &mm.getMetricSnapshot(lockGuard, 60 * 60s); ASSERT_VALUE(12, *snap, "test.set1.set1.count1"); ASSERT_VALUE(24, *snap, "test.set1.set1.countSum"); @@ -239,7 +236,7 @@ TEST_F(SnapshotTest, test_snapshot_two_days) ASSERT_VALUE(1, *snap, "test.set1.set1.averageSum"); // 1 day snapshot - snap = &mm.getMetricSnapshot(lockGuard, 24 * 60 * 60); + snap = &mm.getMetricSnapshot(lockGuard, 24 * 60 * 60s); ASSERT_VALUE(288, *snap, "test.set1.set1.count1"); ASSERT_VALUE(576, *snap, "test.set1.set1.countSum"); diff --git a/metrics/src/tests/summetrictest.cpp b/metrics/src/tests/summetrictest.cpp index 09495ff038d..8e988b65b96 100644 --- a/metrics/src/tests/summetrictest.cpp +++ b/metrics/src/tests/summetrictest.cpp @@ -104,8 +104,7 @@ TEST(SumMetricTest, test_remove) TEST(SumMetricTest, test_start_value) { MetricSnapshot snapshot("active"); - SumMetric<LongValueMetric> sum("foo", {}, "foodesc", - &snapshot.getMetrics()); + SumMetric<LongValueMetric> sum("foo", {}, "foodesc", &snapshot.getMetrics()); LongValueMetric start("start", {}, "", 0); start.set(50); sum.setStartValue(start); @@ -115,7 +114,7 @@ TEST(SumMetricTest, test_start_value) MetricSnapshot copy("copy"); copy.recreateSnapshot(snapshot.getMetrics(), true); - snapshot.addToSnapshot(copy, 100); + snapshot.addToSnapshot(copy, system_time(100s)); LongValueMetric value("value", {}, "", &snapshot.getMetrics()); sum.addMetricToSum(value); diff --git a/metrics/src/vespa/metrics/CMakeLists.txt b/metrics/src/vespa/metrics/CMakeLists.txt index 2441bf95c0b..62254a8d620 100644 --- a/metrics/src/vespa/metrics/CMakeLists.txt +++ b/metrics/src/vespa/metrics/CMakeLists.txt @@ -18,7 +18,6 @@ vespa_add_library(metrics updatehook.cpp valuemetric.cpp valuemetricvalues.cpp - xmlwriter.cpp $<TARGET_OBJECTS:metrics_common> INSTALL lib64 diff --git a/metrics/src/vespa/metrics/jsonwriter.cpp b/metrics/src/vespa/metrics/jsonwriter.cpp index c0d227b8f5a..1b9df3988e1 100644 --- a/metrics/src/vespa/metrics/jsonwriter.cpp +++ b/metrics/src/vespa/metrics/jsonwriter.cpp @@ -23,12 +23,12 @@ JsonWriter::visitSnapshot(const MetricSnapshot& snapshot) { _stream << Object() << "snapshot" << Object() - << "from" << snapshot.getFromTime() - << "to" << snapshot.getToTime() + << "from" << vespalib::count_s(snapshot.getFromTime().time_since_epoch()) + << "to" << vespalib::count_s(snapshot.getToTime().time_since_epoch()) << End() << "values" << Array(); _flag = SNAPSHOT_STARTED; - _period = snapshot.getPeriod(); + _period = vespalib::count_s(snapshot.getPeriod()); // Only prints second resolution return true; } diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp index a0e44ddbeac..2f6fe4c6ba6 100644 --- a/metrics/src/vespa/metrics/metricmanager.cpp +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -10,8 +10,8 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/time.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/stllike/hashtable.hpp> #include <vespa/config/subscription/configsubscriber.hpp> +#include <cinttypes> #include <set> #include <sstream> #include <cassert> @@ -22,36 +22,44 @@ LOG_SETUP(".metrics.manager"); namespace metrics { using Config = MetricsmanagerConfig; +using vespalib::IllegalStateException; +using vespalib::IllegalArgumentException; +using vespalib::make_string_short::fmt; +using vespalib::count_ms; +using vespalib::count_s; +using vespalib::from_s; +using vespalib::to_s; +using vespalib::to_string; MetricManager::ConsumerSpec::ConsumerSpec() = default; MetricManager::ConsumerSpec::~ConsumerSpec() = default; -time_t +time_point MetricManager::Timer::getTime() const { - return vespalib::count_s(vespalib::system_clock::now().time_since_epoch()); + return vespalib::system_clock::now(); } void MetricManager::assertMetricLockLocked(const MetricLockGuard& g) const { if ( ! g.owns(_waiter)) { - throw vespalib::IllegalArgumentException("Given lock does not lock the metric lock.", VESPA_STRLOC); + throw IllegalArgumentException("Given lock does not lock the metric lock.", VESPA_STRLOC); } } -void -MetricManager::ConsumerSpec::print(std::ostream& out, bool verbose, - const std::string& indent) const +vespalib::string +MetricManager::ConsumerSpec::toString() const { - (void) verbose; + vespalib::asciistream out; out << "ConsumerSpec("; std::set<Metric::String> sortedMetrics; for (const Metric::String & name : includedMetrics) { sortedMetrics.insert(name); } for (const auto & s : sortedMetrics) { - out << "\n" << indent << " " << s; + out << "\n" << " " << s; } out << ")"; + return out.str(); } void @@ -63,6 +71,10 @@ MetricManager::ConsumerSpec::addMemoryUsage(MemoryConsumption& mc) const } } +MetricManager::MetricManager() + : MetricManager(std::make_unique<Timer>()) +{ } + MetricManager::MetricManager(std::unique_ptr<Timer> timer) : _activeMetrics("Active metrics showing updates since last snapshot"), _configSubscriber(), @@ -70,11 +82,9 @@ MetricManager::MetricManager(std::unique_ptr<Timer> timer) _config(), _consumerConfig(), _snapshots(), - _totalMetrics(std::make_shared<MetricSnapshot>( - "Empty metrics before init", 0, _activeMetrics.getMetrics(), - false)), + _totalMetrics(std::make_shared<MetricSnapshot>("Empty metrics before init", 0s, _activeMetrics.getMetrics(), false)), _timer(std::move(timer)), - _lastProcessedTime(0), + _lastProcessedTime(), _snapshotUnsetMetrics(false), _consumerConfigChanged(false), _metricManagerMetrics("metricmanager", {}, "Metrics for the metric manager upkeep tasks", nullptr), @@ -108,29 +118,26 @@ MetricManager::stop() } void -MetricManager::addMetricUpdateHook(UpdateHook& hook, uint32_t period) +MetricManager::addMetricUpdateHook(UpdateHook& hook) { - hook._period = period; + hook.updateNextCall(_timer->getTime()); std::lock_guard sync(_waiter); - // If we've already initialized manager, log period has been set. - // In this case. Call first time after period - hook._nextCall = _timer->getTime() + period; - if (period == 0) { - for (UpdateHook * sHook : _snapshotUpdateHooks) { - if (sHook == &hook) { + if ( hook.is_periodic() ) { + for (UpdateHook * pHook : _periodicUpdateHooks) { + if (pHook == &hook) { LOG(warning, "Update hook already registered"); return; } } - _snapshotUpdateHooks.push_back(&hook); + _periodicUpdateHooks.push_back(&hook); } else { - for (UpdateHook * pHook : _periodicUpdateHooks) { - if (pHook == &hook) { + for (UpdateHook * sHook : _snapshotUpdateHooks) { + if (sHook == &hook) { LOG(warning, "Update hook already registered"); return; } } - _periodicUpdateHooks.push_back(&hook); + _snapshotUpdateHooks.push_back(&hook); } } @@ -138,17 +145,17 @@ void MetricManager::removeMetricUpdateHook(UpdateHook& hook) { std::lock_guard sync(_waiter); - if (hook._period == 0) { - for (auto it = _snapshotUpdateHooks.begin(); it != _snapshotUpdateHooks.end(); it++) { + if (hook.is_periodic()) { + for (auto it = _periodicUpdateHooks.begin(); it != _periodicUpdateHooks.end(); it++) { if (*it == &hook) { - _snapshotUpdateHooks.erase(it); + _periodicUpdateHooks.erase(it); return; } } } else { - for (auto it = _periodicUpdateHooks.begin(); it != _periodicUpdateHooks.end(); it++) { + for (auto it = _snapshotUpdateHooks.begin(); it != _snapshotUpdateHooks.end(); it++) { if (*it == &hook) { - _periodicUpdateHooks.erase(it); + _snapshotUpdateHooks.erase(it); return; } } @@ -165,9 +172,8 @@ void MetricManager::init(const config::ConfigUri & uri, bool startThread) { if (isInitialized()) { - throw vespalib::IllegalStateException( - "The metric manager have already been initialized. " - "It can only be initialized once.", VESPA_STRLOC); + throw IllegalStateException("The metric manager have already been initialized. " + "It can only be initialized once.", VESPA_STRLOC); } LOG(debug, "Initializing metric manager."); _configSubscriber = std::make_unique<config::ConfigSubscriber>(uri.getContext()); @@ -180,7 +186,7 @@ MetricManager::init(const config::ConfigUri & uri, bool startThread) // Wait for first iteration to have completed, such that it is safe // to access snapshots afterwards. MetricLockGuard sync(_waiter); - while (_lastProcessedTime.load(std::memory_order_relaxed) == 0) { + while (_lastProcessedTime.load(std::memory_order_relaxed) == time_point()) { _cond.wait_for(sync, 1ms); } } else { @@ -233,8 +239,10 @@ struct ConsumerMetricBuilder : public MetricVisitor { bool nameRemoved; uint32_t metricCount; - Result() : tagAdded(false), tagRemoved(false), - nameAdded(false), nameRemoved(false), metricCount(0) {} + Result() + : tagAdded(false), tagRemoved(false), + nameAdded(false), nameRemoved(false), metricCount(0) + {} }; std::list<Result> result; @@ -353,9 +361,7 @@ ConsumerMetricBuilder::~ConsumerMetricBuilder() = default; void MetricManager::checkMetricsAltered(const MetricLockGuard & guard) { - if (_activeMetrics.getMetrics().isRegistrationAltered() - || _consumerConfigChanged) - { + if (_activeMetrics.getMetrics().isRegistrationAltered() || _consumerConfigChanged) { handleMetricsAltered(guard); } } @@ -376,17 +382,17 @@ MetricManager::handleMetricsAltered(const MetricLockGuard & guard) LOG(info, "Metrics registration changes detected. Handling changes."); } _activeMetrics.getMetrics().clearRegistrationAltered(); - std::map<Metric::String, ConsumerSpec::SP> configMap; + std::map<Metric::String, ConsumerSpec> configMap; LOG(debug, "Calculating new consumer config"); for (const auto & consumer : _config->consumer) { ConsumerMetricBuilder consumerMetricBuilder(consumer); _activeMetrics.getMetrics().visit(consumerMetricBuilder); - configMap[consumer.name] = std::make_shared<ConsumerSpec>(std::move(consumerMetricBuilder._matchedMetrics)); + configMap[consumer.name] = ConsumerSpec(std::move(consumerMetricBuilder._matchedMetrics)); } LOG(debug, "Recreating snapshots to include altered metrics"); _totalMetrics->recreateSnapshot(_activeMetrics.getMetrics(), _snapshotUnsetMetrics); - for (uint32_t i=0; i<_snapshots.size(); ++i) { - _snapshots[i]->recreateSnapshot(_activeMetrics.getMetrics(), _snapshotUnsetMetrics); + for (const auto & snapshot: _snapshots) { + snapshot->recreateSnapshot(_activeMetrics.getMetrics(), _snapshotUnsetMetrics); } LOG(debug, "Setting new consumer config. Clearing dirty flag"); _consumerConfig.swap(configMap); @@ -394,24 +400,25 @@ MetricManager::handleMetricsAltered(const MetricLockGuard & guard) } namespace { - bool setSnapshotName(std::ostream& out, const char* name, uint32_t length, uint32_t period) - { - if (length % period != 0) return false; - out << (length / period) << ' ' << name; - if (length / period != 1) out << "s"; - return true; - } + +bool +setSnapshotName(std::ostream& out, const char* name, uint32_t length, uint32_t period) { + if (length % period != 0) return false; + out << (length / period) << ' ' << name; + if (length / period != 1) out << "s"; + return true; +} + } std::vector<MetricManager::SnapSpec> MetricManager::createSnapshotPeriods(const Config& config) { std::vector<SnapSpec> result; - try{ - for (uint32_t i=0; i<config.snapshot.periods.size(); ++i) { - uint32_t length = config.snapshot.periods[i]; + try { + for (auto length : config.snapshot.periods) { if (length < 1) - throw vespalib::IllegalStateException("Snapshot periods must be positive numbers", VESPA_STRLOC); + throw IllegalStateException("Snapshot periods must be positive numbers", VESPA_STRLOC); std::ostringstream name; if (setSnapshotName(name, "week", length, 60 * 60 * 24 * 7)) { } else if (setSnapshotName(name, "day", length, 60 * 60 * 24)) { @@ -420,28 +427,25 @@ MetricManager::createSnapshotPeriods(const Config& config) } else { name << length << " seconds"; } - result.push_back(SnapSpec(length, name.str())); + result.emplace_back(vespalib::from_s<time_point::duration>(length), name.str()); } for (uint32_t i=1; i<result.size(); ++i) { - if (result[i].first % result[i-1].first != 0) { + if (result[i].first % result[i-1].first != vespalib::duration::zero()) { std::ostringstream ost; - ost << "Period " << result[i].first - << " is not a multiplum of period " + ost << "Period " << result[i].first << " is not a multiplum of period " << result[i-1].first << " which is needs to be."; - throw vespalib::IllegalStateException( - ost.str(), VESPA_STRLOC); + throw IllegalStateException(ost.str(), VESPA_STRLOC); } } } catch (vespalib::Exception& e) { - LOG(warning, "Invalid snapshot periods specified. Using defaults: %s", - e.getMessage().c_str()); + LOG(warning, "Invalid snapshot periods specified. Using defaults: %s", e.getMessage().c_str()); result.clear(); } if (result.empty()) { - result.push_back(SnapSpec(60 * 5, "5 minute")); - result.push_back(SnapSpec(60 * 60, "1 hour")); - result.push_back(SnapSpec(60 * 60 * 24, "1 day")); - result.push_back(SnapSpec(60 * 60 * 24 * 7, "1 week")); + result.emplace_back(60s * 5, "5 minute"); + result.emplace_back(60s * 60, "1 hour"); + result.emplace_back(60s * 60 * 24, "1 day"); + result.emplace_back(60s * 60 * 24 * 7, "1 week"); } return result; } @@ -454,30 +458,23 @@ MetricManager::configure(const MetricLockGuard & , std::unique_ptr<Config> confi std::ostringstream ost; config::OstreamConfigWriter w(ost); w.write(*config); - LOG(debug, "Received new config for metric manager: %s", - ost.str().c_str()); + LOG(debug, "Received new config for metric manager: %s", ost.str().c_str()); } if (_snapshots.empty()) { LOG(debug, "Initializing snapshots as this is first configure call"); std::vector<SnapSpec> snapshotPeriods(createSnapshotPeriods(*config)); - // Set up snapshots only first time. We don't allow live reconfig - // of snapshot periods. - time_t currentTime(_timer->getTime()); + // Set up snapshots only first time. We don't allow live reconfig + // of snapshot periods. + time_point currentTime = _timer->getTime(); _activeMetrics.setFromTime(currentTime); uint32_t count = 1; - for (uint32_t i = 0; i< snapshotPeriods.size(); ++i) - { + for (uint32_t i = 0; i< snapshotPeriods.size(); ++i) { uint32_t nextCount = 1; if (i + 1 < snapshotPeriods.size()) { - nextCount = snapshotPeriods[i + 1].first - / snapshotPeriods[i].first; - if (snapshotPeriods[i + 1].first - % snapshotPeriods[i].first != 0) - { - throw vespalib::IllegalStateException( - "Snapshot periods must be multiplum of each other", - VESPA_STRLOC); + nextCount = snapshotPeriods[i + 1].first / snapshotPeriods[i].first; + if ((snapshotPeriods[i + 1].first % snapshotPeriods[i].first) != time_point::duration::zero()) { + throw IllegalStateException("Snapshot periods must be multiplum of each other",VESPA_STRLOC); } } _snapshots.push_back(std::make_shared<MetricSnapshotSet>( @@ -485,13 +482,11 @@ MetricManager::configure(const MetricLockGuard & , std::unique_ptr<Config> confi _activeMetrics.getMetrics(), _snapshotUnsetMetrics)); count = nextCount; } - // Add all time snapshot. - _totalMetrics = std::make_shared<MetricSnapshot>("All time snapshot", 0, _activeMetrics.getMetrics(), _snapshotUnsetMetrics); + // Add all time snapshot. + _totalMetrics = std::make_shared<MetricSnapshot>("All time snapshot", 0s, _activeMetrics.getMetrics(), _snapshotUnsetMetrics); _totalMetrics->reset(currentTime); } - if (_config.get() == 0 - || _config->consumer.size() != config->consumer.size()) - { + if (_config.get() == 0 || (_config->consumer.size() != config->consumer.size())) { _consumerConfigChanged = true; } else { for (uint32_t i=0; i<_config->consumer.size(); ++i) { @@ -508,94 +503,66 @@ MetricManager::configure(const MetricLockGuard & , std::unique_ptr<Config> confi } -MetricManager::ConsumerSpec::SP +const MetricManager::ConsumerSpec * MetricManager::getConsumerSpec(const MetricLockGuard &, const Metric::String& consumer) const { auto it(_consumerConfig.find(consumer)); - return (it != _consumerConfig.end() ? it->second : ConsumerSpec::SP()); + return (it != _consumerConfig.end() ? &it->second : nullptr); } -//#define VERIFY_ALL_METRICS_VISITED 1 namespace { - struct ConsumerMetricVisitor : public MetricVisitor { - const MetricManager::ConsumerSpec& _metricsToMatch; - MetricVisitor& _client; -#ifdef VERIFY_ALL_METRICS_VISITED - std::set<Metric::String> _visitedMetrics; -#endif - - ConsumerMetricVisitor(const MetricManager::ConsumerSpec& spec, - MetricVisitor& clientVisitor) - : _metricsToMatch(spec), _client(clientVisitor) {} - - bool visitMetricSet(const MetricSet& metricSet, - bool autoGenerated) override - { - if (metricSet.isTopSet()) return true; - return (_metricsToMatch.contains(metricSet) - && _client.visitMetricSet(metricSet, autoGenerated)); - } - void doneVisitingMetricSet(const MetricSet& metricSet) override { - if (!metricSet.isTopSet()) { -#ifdef VERIFY_ALL_METRICS_VISITED - _visitedMetrics.insert(metricSet.getPath()); -#endif - _client.doneVisitingMetricSet(metricSet); - } +struct ConsumerMetricVisitor : public MetricVisitor { + const MetricManager::ConsumerSpec& _metricsToMatch; + MetricVisitor& _client; + + ConsumerMetricVisitor(const MetricManager::ConsumerSpec& spec, MetricVisitor& clientVisitor) + : _metricsToMatch(spec), + _client(clientVisitor) + { } + + bool visitMetricSet(const MetricSet& metricSet, bool autoGenerated) override { + if (metricSet.isTopSet()) return true; + return (_metricsToMatch.contains(metricSet) + && _client.visitMetricSet(metricSet, autoGenerated)); + } + void doneVisitingMetricSet(const MetricSet& metricSet) override { + if (!metricSet.isTopSet()) { + _client.doneVisitingMetricSet(metricSet); } - bool visitCountMetric(const AbstractCountMetric& metric, - bool autoGenerated) override - { - if (_metricsToMatch.contains(metric)) { -#ifdef VERIFY_ALL_METRICS_VISITED - _visitedMetrics.insert(metric.getPath()); -#endif - return _client.visitCountMetric(metric, autoGenerated); - } - return true; + } + bool visitCountMetric(const AbstractCountMetric& metric, bool autoGenerated) override { + if (_metricsToMatch.contains(metric)) { + return _client.visitCountMetric(metric, autoGenerated); } - bool visitValueMetric(const AbstractValueMetric& metric, - bool autoGenerated) override - { - if (_metricsToMatch.contains(metric)) { -#ifdef VERIFY_ALL_METRICS_VISITED - _visitedMetrics.insert(metric.getPath()); -#endif - return _client.visitValueMetric(metric, autoGenerated); - } - return true; + return true; + } + bool visitValueMetric(const AbstractValueMetric& metric, bool autoGenerated) override { + if (_metricsToMatch.contains(metric)) { + return _client.visitValueMetric(metric, autoGenerated); } - }; + return true; + } +}; } void -MetricManager::visit(const MetricLockGuard & guard, const MetricSnapshot& snapshot, MetricVisitor& visitor, - const std::string& consumer) const +MetricManager::visit(const MetricLockGuard & guard, const MetricSnapshot& snapshot, + MetricVisitor& visitor, const std::string& consumer) const { if (visitor.visitSnapshot(snapshot)) { if (consumer == "") { snapshot.getMetrics().visit(visitor); } else { - ConsumerSpec::SP consumerSpec(getConsumerSpec(guard, consumer)); - if (consumerSpec.get()) { + const ConsumerSpec * consumerSpec = getConsumerSpec(guard, consumer); + if (consumerSpec) { + ConsumerMetricVisitor consumerVis(*consumerSpec, visitor); snapshot.getMetrics().visit(consumerVis); -#ifdef VERIFY_ALL_METRICS_VISITED - for (auto metric = consumerSpec->includedMetrics) { - if (consumerVis._visitedMetrics.find(metric) - == consumerVis._visitedMetrics.end()) - { - LOG(debug, "Failed to find metric %s to be visited.", metric.c_str()); - } - } -#endif } else { - LOGBP(debug, - "Requested metrics for non-defined consumer '%s'.", - consumer.c_str()); + LOGBP(debug, "Requested metrics for non-defined consumer '%s'.", consumer.c_str()); } } visitor.doneVisitingSnapshot(snapshot); @@ -603,52 +570,45 @@ MetricManager::visit(const MetricLockGuard & guard, const MetricSnapshot& snapsh visitor.doneVisiting(); } -std::vector<uint32_t> +std::vector<time_point::duration> MetricManager::getSnapshotPeriods(const MetricLockGuard& l) const { assertMetricLockLocked(l); - std::vector<uint32_t> result(_snapshots.size()); - for (uint32_t i=0; i<_snapshots.size(); ++i) { - result[i] = _snapshots[i]->getPeriod(); + std::vector<time_point::duration> result; + result.reserve(_snapshots.size()); + for (const auto & snapshot : _snapshots) { + result.emplace_back(snapshot->getPeriod()); } return result; } // Client should have grabbed metrics lock before doing this const MetricSnapshot& -MetricManager::getMetricSnapshot(const MetricLockGuard& l, - uint32_t period, bool getInProgressSet) const +MetricManager::getMetricSnapshot(const MetricLockGuard& l, vespalib::duration period, bool getInProgressSet) const { assertMetricLockLocked(l); - for (uint32_t i=0; i<_snapshots.size(); ++i) { - if (_snapshots[i]->getPeriod() == period) { - if (_snapshots[i]->getCount() == 1 && getInProgressSet) { - throw vespalib::IllegalStateException( - "No temporary snapshot for set " - + _snapshots[i]->getName(), VESPA_STRLOC); + for (const auto & snapshot : _snapshots) { + if (snapshot->getPeriod() == period) { + if (snapshot->getCount() == 1 && getInProgressSet) { + throw IllegalStateException("No temporary snapshot for set " + snapshot->getName(), VESPA_STRLOC); } - return _snapshots[i]->getSnapshot(getInProgressSet); + return snapshot->getSnapshot(getInProgressSet); } } - std::ostringstream ost; - ost << "No snapshot for period of length " << period << " exist."; - throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); + throw IllegalArgumentException(fmt("No snapshot for period of length %f exist.", vespalib::to_s(period)), VESPA_STRLOC); } // Client should have grabbed metrics lock before doing this const MetricSnapshotSet& -MetricManager::getMetricSnapshotSet(const MetricLockGuard& l, - uint32_t period) const +MetricManager::getMetricSnapshotSet(const MetricLockGuard& l, vespalib::duration period) const { assertMetricLockLocked(l); - for (uint32_t i=0; i<_snapshots.size(); ++i) { - if (_snapshots[i]->getPeriod() == period) { - return *_snapshots[i]; + for (const auto & snapshot : _snapshots) { + if (snapshot->getPeriod() == period) { + return *snapshot; } } - std::ostringstream ost; - ost << "No snapshot set for period of length " << period << " exist."; - throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); + throw IllegalArgumentException(fmt("No snapshot set for period of length %f exist.", to_s(period)), VESPA_STRLOC); } void @@ -659,51 +619,46 @@ MetricManager::timeChangedNotification() const } void -MetricManager::updateMetrics(bool includeSnapshotOnlyHooks) +MetricManager::updateMetrics() { - LOG(debug, "Calling metric update hooks%s.", - includeSnapshotOnlyHooks ? ", including snapshot hooks" : ""); - // Ensure we're not in the way of the background thread + // Ensure we're not in the way of the background thread MetricLockGuard sync(_waiter); - LOG(debug, "Giving %zu periodic update hooks.", _periodicUpdateHooks.size()); - updatePeriodicMetrics(sync, 0, true); - if (includeSnapshotOnlyHooks) { - LOG(debug, "Giving %zu snapshot update hooks.", _snapshotUpdateHooks.size()); - updateSnapshotMetrics(sync); - } + LOG(debug, "Calling %zu periodic update hooks.", _periodicUpdateHooks.size()); + updatePeriodicMetrics(sync, time_point(), true); + updateSnapshotMetrics(sync); } // When this is called, the thread monitor lock has already been grabbed -time_t -MetricManager::updatePeriodicMetrics(const MetricLockGuard & guard, time_t updateTime, bool outOfSchedule) +time_point +MetricManager::updatePeriodicMetrics(const MetricLockGuard & guard, time_point updateTime, bool outOfSchedule) { - time_t nextUpdateTime = std::numeric_limits<time_t>::max(); - time_t preTime = _timer->getTimeInMilliSecs(); + assertMetricLockLocked(guard); + time_point nextUpdateTime = time_point::max(); + time_point preTime = _timer->getTimeInMilliSecs(); for (auto hook : _periodicUpdateHooks) { - if (hook->_nextCall <= updateTime) { + if (hook->expired(updateTime)) { hook->updateMetrics(guard); - if (hook->_nextCall + hook->_period < updateTime) { - if (hook->_nextCall != 0) { - LOG(debug, "Updated hook %s at time %" PRIu64 ", but next " - "run in %u seconds have already passed as time" - " is %" PRIu64 ". Bumping next call to current " - "time + period.", - hook->_name, static_cast<uint64_t>(hook->_nextCall), hook->_period, static_cast<uint64_t>(updateTime)); + if (hook->expired(updateTime - hook->getPeriod())) { + if (hook->has_valid_expiry()) { + LOG(debug, "Updated hook %s at time %s, but next run in %" PRId64 " seconds have already passed as " + "time is %s. Bumping next call to current time + period.", + hook->getName(), to_string(hook->getNextCall()).c_str(), + count_s(hook->getPeriod()), to_string(updateTime).c_str()); } - hook->_nextCall = updateTime + hook->_period; + hook->updateNextCall(updateTime); } else { - hook->_nextCall += hook->_period; + hook->updateNextCall(); } - time_t postTime = _timer->getTimeInMilliSecs(); - _periodicHookLatency.addValue(postTime - preTime); + time_point postTime = _timer->getTimeInMilliSecs(); + _periodicHookLatency.addValue(count_ms(postTime - preTime)); preTime = postTime; } else if (outOfSchedule) { hook->updateMetrics(guard); - time_t postTime = _timer->getTimeInMilliSecs(); - _periodicHookLatency.addValue(postTime - preTime); + time_point postTime = _timer->getTimeInMilliSecs(); + _periodicHookLatency.addValue(count_ms(postTime - preTime)); preTime = postTime; } - nextUpdateTime = std::min(nextUpdateTime, hook->_nextCall); + nextUpdateTime = std::min(nextUpdateTime, hook->getNextCall()); } return nextUpdateTime; } @@ -712,11 +667,12 @@ MetricManager::updatePeriodicMetrics(const MetricLockGuard & guard, time_t updat void MetricManager::updateSnapshotMetrics(const MetricLockGuard & guard) { - time_t preTime = _timer->getTimeInMilliSecs(); - for (auto it = _snapshotUpdateHooks.begin(); it != _snapshotUpdateHooks.end(); ++it) { - (**it).updateMetrics(guard); - time_t postTime = _timer->getTimeInMilliSecs(); - _snapshotHookLatency.addValue(postTime - preTime); + assertMetricLockLocked(guard); + time_point preTime = _timer->getTimeInMilliSecs(); + for (const auto & hook : _snapshotUpdateHooks) { + hook->updateMetrics(guard); + time_point postTime = _timer->getTimeInMilliSecs(); + _snapshotHookLatency.addValue(count_ms(postTime - preTime)); preTime = postTime; } } @@ -731,19 +687,19 @@ MetricManager::forceEventLogging() } void -MetricManager::reset(time_t currentTime) +MetricManager::reset(system_time currentTime) { - time_t preTime = _timer->getTimeInMilliSecs(); + time_point preTime = _timer->getTimeInMilliSecs(); // Resetting implies visiting metrics, which needs to grab metric lock // to avoid conflict with adding/removal of metrics std::lock_guard waiterLock(_waiter); _activeMetrics.reset(currentTime); - for (uint32_t i=0; i<_snapshots.size(); ++i) { - _snapshots[i]->reset(currentTime); + for (const auto & snapshot : _snapshots) { + snapshot->reset(currentTime); } _totalMetrics->reset(currentTime); - time_t postTime = _timer->getTimeInMilliSecs(); - _resetLatency.addValue(postTime - preTime); + time_point postTime = _timer->getTimeInMilliSecs(); + _resetLatency.addValue(count_ms(postTime - preTime)); } void @@ -755,36 +711,35 @@ MetricManager::run() // we constantly add next time to do something from the last timer. // For that to work, we need to initialize timers on first iteration // to set them to current time. - time_t currentTime = _timer->getTime(); + system_time currentTime = _timer->getTime(); for (auto & snapshot : _snapshots) { snapshot->setFromTime(currentTime); } for (auto hook : _periodicUpdateHooks) { - hook->_nextCall = currentTime; + hook->setNextCall(currentTime); } // Ensure correct time for first snapshot _snapshots[0]->getSnapshot().setToTime(currentTime); while (!stop_requested()) { currentTime = _timer->getTime(); - time_t next = tick(sync, currentTime); + time_point next = tick(sync, currentTime); if (currentTime < next) { - size_t ms = (next - currentTime) * 1000; - _cond.wait_for(sync, std::chrono::milliseconds(ms)); - _sleepTimes.addValue(ms); + vespalib::duration wait_time = next - currentTime; + _cond.wait_for(sync, wait_time); + _sleepTimes.addValue(count_ms(wait_time)); } else { _sleepTimes.addValue(0); } } } -time_t -MetricManager::tick(const MetricLockGuard & guard, time_t currentTime) +time_point +MetricManager::tick(const MetricLockGuard & guard, time_point currentTime) { - LOG(spam, "Worker thread starting to process for time %" PRIu64 ".", - static_cast<uint64_t>(currentTime)); + LOG(spam, "Worker thread starting to process for time %s.", to_string(currentTime).c_str()); // Check for new config and reconfigure - if (_configSubscriber.get() && _configSubscriber->nextConfigNow()) { + if (_configSubscriber && _configSubscriber->nextConfigNow()) { configure(guard, _configHandle->getConfig()); } @@ -794,8 +749,8 @@ MetricManager::tick(const MetricLockGuard & guard, time_t currentTime) checkMetricsAltered(guard); // Set next work time to the time we want to take next snapshot. - time_t nextWorkTime = _snapshots[0]->getToTime() + _snapshots[0]->getPeriod(); - time_t nextUpdateHookTime; + time_point nextWorkTime = _snapshots[0]->getNextWorkTime(); + time_point nextUpdateHookTime; if (nextWorkTime <= currentTime) { // If taking a new snapshot or logging, force calls to all // update hooks. @@ -811,96 +766,82 @@ MetricManager::tick(const MetricLockGuard & guard, time_t currentTime) // Do snapshotting if it is time if (nextWorkTime <= currentTime) takeSnapshots(guard, nextWorkTime); - _lastProcessedTime.store(nextWorkTime <= currentTime ? nextWorkTime : currentTime, - std::memory_order_relaxed); - LOG(spam, "Worker thread done with processing for time %" PRIu64 ".", - static_cast<uint64_t>(_lastProcessedTime.load(std::memory_order_relaxed))); - time_t next = _snapshots[0]->getPeriod() + _snapshots[0]->getToTime(); + _lastProcessedTime.store((nextWorkTime <= currentTime ? nextWorkTime : currentTime), std::memory_order_relaxed); + LOG(spam, "Worker thread done with processing for time %s.", + to_string(_lastProcessedTime.load(std::memory_order_relaxed)).c_str()); + time_point next = _snapshots[0]->getNextWorkTime(); next = std::min(next, nextUpdateHookTime); return next; } void -MetricManager::takeSnapshots(const MetricLockGuard &, time_t timeToProcess) +MetricManager::takeSnapshots(const MetricLockGuard & guard, system_time timeToProcess) { + assertMetricLockLocked(guard); // If not time to do dump data from active snapshot yet, nothing to do if (!_snapshots[0]->timeForAnotherSnapshot(timeToProcess)) { - LOG(spam, "Not time to process snapshot %s at time %" PRIu64 ". Current " - "first period (%u) snapshot goes from %" PRIu64 " to %" PRIu64, - _snapshots[0]->getName().c_str(), static_cast<uint64_t>(timeToProcess), - _snapshots[0]->getPeriod(), static_cast<uint64_t>(_snapshots[0]->getFromTime()), - static_cast<uint64_t>(_snapshots[0]->getToTime())); + LOG(spam, "Not time to process snapshot %s at time %s. Current " + "first period (%f) snapshot goes from %s to %s", + _snapshots[0]->getName().c_str(), to_string(timeToProcess).c_str(), + to_s(_snapshots[0]->getPeriod()), to_string(_snapshots[0]->getFromTime()).c_str(), + to_string(_snapshots[0]->getToTime()).c_str()); return; } - time_t preTime = _timer->getTimeInMilliSecs(); - LOG(debug, "Updating %s snapshot and total metrics at time %" PRIu64 ".", - _snapshots[0]->getName().c_str(), static_cast<uint64_t>(timeToProcess)); + time_point preTime = _timer->getTimeInMilliSecs(); + LOG(debug, "Updating %s snapshot and total metrics at time %s.", + _snapshots[0]->getName().c_str(), to_string(timeToProcess).c_str()); MetricSnapshot& firstTarget(_snapshots[0]->getNextTarget()); firstTarget.reset(_activeMetrics.getFromTime()); _activeMetrics.addToSnapshot(firstTarget, false, timeToProcess); _activeMetrics.addToSnapshot(*_totalMetrics, false, timeToProcess); _activeMetrics.reset(timeToProcess); - LOG(debug, "After snapshotting, " - "active metrics goes from %" PRIu64 " to %" PRIu64", " - "and 5 minute metrics goes from %" PRIu64 " to %" PRIu64".", - static_cast<uint64_t>(_activeMetrics.getFromTime()), static_cast<uint64_t>(_activeMetrics.getToTime()), - static_cast<uint64_t>(firstTarget.getFromTime()), static_cast<uint64_t>(firstTarget.getToTime())); + LOG(debug, "After snapshotting, active metrics goes from %s to %s, and 5 minute metrics goes from %s to %s.", + to_string(_activeMetrics.getFromTime()).c_str(), to_string(_activeMetrics.getToTime()).c_str(), + to_string(firstTarget.getFromTime()).c_str(), to_string(firstTarget.getToTime()).c_str()); // Update later snapshots if it is time for it for (uint32_t i=1; i<_snapshots.size(); ++i) { - LOG(debug, "Adding data from last snapshot to building snapshot of " - "next period snapshot %s.", + LOG(debug, "Adding data from last snapshot to building snapshot of next period snapshot %s.", _snapshots[i]->getName().c_str()); MetricSnapshot& target(_snapshots[i]->getNextTarget()); - _snapshots[i-1]->getSnapshot().addToSnapshot( - target, false, timeToProcess); + _snapshots[i-1]->getSnapshot().addToSnapshot(target, false, timeToProcess); target.setToTime(timeToProcess); if (!_snapshots[i]->haveCompletedNewPeriod(timeToProcess)) { - LOG(debug, "Not time to roll snapshot %s yet. %u of %u snapshot " - "taken at time %" PRIu64 ", and period of %u is not up " - "yet as we're currently processing for time %" PRIu64 ".", - _snapshots[i]->getName().c_str(), - _snapshots[i]->getBuilderCount(), - _snapshots[i]->getCount(), - static_cast<uint64_t> - (_snapshots[i]->getBuilderCount() * _snapshots[i]->getPeriod() - + _snapshots[i]->getFromTime()), - _snapshots[i]->getPeriod(), - static_cast<uint64_t>(timeToProcess)); + LOG(debug, "Not time to roll snapshot %s yet. %u of %u snapshot taken at time %s, and period of %f " + "is not up yet as we're currently processing for time %s.", + _snapshots[i]->getName().c_str(), _snapshots[i]->getBuilderCount(), _snapshots[i]->getCount(), + to_string(_snapshots[i]->getBuilderCount() * _snapshots[i]->getPeriod() + _snapshots[i]->getFromTime()).c_str(), + to_s(_snapshots[i]->getPeriod()), to_string(timeToProcess).c_str()); break; } else { - LOG(debug, "Rolled snapshot %s at time %" PRIu64 ".", - _snapshots[i]->getName().c_str(), - static_cast<uint64_t>(timeToProcess)); + LOG(debug, "Rolled snapshot %s at time %s.", + _snapshots[i]->getName().c_str(), to_string(timeToProcess).c_str()); } } - time_t postTime = _timer->getTimeInMilliSecs(); - _snapshotLatency.addValue(postTime - preTime); + time_point postTime = _timer->getTimeInMilliSecs(); + _snapshotLatency.addValue(count_ms(postTime - preTime)); } MemoryConsumption::UP MetricManager::getMemoryConsumption(const MetricLockGuard & guard) const { - (void) guard; - MemoryConsumption::UP mc(new MemoryConsumption); + assertMetricLockLocked(guard); + auto mc = std::make_unique<MemoryConsumption>(); mc->_consumerCount += _consumerConfig.size(); - mc->_consumerMeta += (sizeof(ConsumerSpec::SP) + sizeof(ConsumerSpec)) - * _consumerConfig.size(); - for (auto it = _consumerConfig.begin(); it != _consumerConfig.end(); ++it) { - mc->_consumerId += mc->getStringMemoryUsage( - it->first, mc->_consumerIdUnique) - + sizeof(Metric::String); - it->second->addMemoryUsage(*mc); + mc->_consumerMeta += sizeof(ConsumerSpec) * _consumerConfig.size(); + for (const auto & consumer : _consumerConfig) { + mc->_consumerId += mc->getStringMemoryUsage(consumer.first, mc->_consumerIdUnique) + sizeof(Metric::String); + consumer.second.addMemoryUsage(*mc); } uint32_t preTotal = mc->getTotalMemoryUsage(); _activeMetrics.addMemoryUsage(*mc); uint32_t postTotal = mc->getTotalMemoryUsage(); mc->addSnapShotUsage("active", postTotal - preTotal); preTotal = postTotal; - for (uint32_t i=0; i<_snapshots.size(); ++i) { - _snapshots[i]->addMemoryUsage(*mc); + for (const auto & snapshot : _snapshots) { + snapshot->addMemoryUsage(*mc); postTotal = mc->getTotalMemoryUsage(); - mc->addSnapShotUsage(_snapshots[i]->getName(), postTotal - preTotal); + mc->addSnapShotUsage(snapshot->getName(), postTotal - preTotal); preTotal = postTotal; } _totalMetrics->addMemoryUsage(*mc); diff --git a/metrics/src/vespa/metrics/metricmanager.h b/metrics/src/vespa/metrics/metricmanager.h index b1777a1d228..6f40e7961f4 100644 --- a/metrics/src/vespa/metrics/metricmanager.h +++ b/metrics/src/vespa/metrics/metricmanager.h @@ -55,8 +55,7 @@ #include <vespa/config/subscription/configuri.h> #include <map> #include <list> - -template class vespalib::hash_set<metrics::Metric::String>; +#include <thread> namespace metrics { @@ -66,29 +65,27 @@ public: struct Timer { virtual ~Timer() = default; - virtual time_t getTime() const; - virtual time_t getTimeInMilliSecs() const { return getTime() * 1000; } + virtual time_point getTime() const; + time_point getTimeInMilliSecs() const { return getTime(); } }; /** * Spec saved from config. If metricSetChildren has content, metric pointed * to is a metric set. */ - struct ConsumerSpec : public vespalib::Printable { - using SP = std::shared_ptr<ConsumerSpec>; - + struct ConsumerSpec { vespalib::hash_set<Metric::String> includedMetrics; + ConsumerSpec(ConsumerSpec &&) noexcept = default; ConsumerSpec & operator= (ConsumerSpec &&) noexcept = default; ConsumerSpec(); - ~ConsumerSpec() override; + ~ConsumerSpec(); bool contains(const Metric& m) const { return (includedMetrics.find(m.getPath()) != includedMetrics.end()); } - void print(std::ostream& out, bool verbose, - const std::string& indent) const override; + vespalib::string toString() const; void addMemoryUsage(MemoryConsumption&) const; }; @@ -98,15 +95,15 @@ private: std::unique_ptr<config::ConfigSubscriber> _configSubscriber; std::unique_ptr<config::ConfigHandle<MetricsmanagerConfig>> _configHandle; std::unique_ptr<MetricsmanagerConfig> _config; - std::map<Metric::String, ConsumerSpec::SP> _consumerConfig; + std::map<Metric::String, ConsumerSpec> _consumerConfig; std::list<UpdateHook*> _periodicUpdateHooks; std::list<UpdateHook*> _snapshotUpdateHooks; mutable std::mutex _waiter; mutable std::condition_variable _cond; - std::vector<MetricSnapshotSet::SP> _snapshots; - MetricSnapshot::SP _totalMetrics; + std::vector<std::shared_ptr<MetricSnapshotSet>> _snapshots; + std::shared_ptr<MetricSnapshot> _totalMetrics; std::unique_ptr<Timer> _timer; - std::atomic<time_t> _lastProcessedTime; + std::atomic<time_point> _lastProcessedTime; // Should be added to config, but wont now due to problems with // upgrading bool _snapshotUnsetMetrics; @@ -125,7 +122,8 @@ private: bool stop_requested() const { return _stop_requested.load(std::memory_order_relaxed); } public: - MetricManager(std::unique_ptr<Timer> timer = std::make_unique<Timer>()); + MetricManager(); + MetricManager(std::unique_ptr<Timer> timer); ~MetricManager(); void stop(); @@ -137,7 +135,7 @@ public: * snapshotting and metric logging, to make the metrics the best as they can * be at those occasions. * - * @param period Period in seconds for how often callback should be called. + * @param period Period for how often callback should be called. * The default value of 0, means only before snapshotting or * logging, while another value will give callbacks each * period seconds. Expensive metrics to calculate will @@ -146,7 +144,7 @@ public: * seconds or so. Any value of period >= the smallest snapshot * time will behave identically as if period is set to 0. */ - void addMetricUpdateHook(UpdateHook&, uint32_t period = 0); + void addMetricUpdateHook(UpdateHook&); /** Remove a metric update hook so it won't get any more updates. */ void removeMetricUpdateHook(UpdateHook&); @@ -156,7 +154,7 @@ public: * nice values before reporting something. * This function can not be called from an update hook callback. */ - void updateMetrics(bool includeSnapshotOnlyHooks = false); + void updateMetrics(); /** * Force event logging to happen now. @@ -190,7 +188,7 @@ public: * Reset all metrics including all snapshots. * This function can not be called from an update hook callback. */ - void reset(time_t currentTime); + void reset(system_time currentTime); /** * Read configuration. Before reading config, all metrics should be set @@ -198,7 +196,10 @@ public: * of consumers. readConfig() will start a config subscription. It should * not be called multiple times. */ - void init(const config::ConfigUri & uri, bool startThread = true); + void init(const config::ConfigUri & uri, bool startThread); + void init(const config::ConfigUri & uri) { + init(uri, true); + } /** * Visit a given snapshot for a given consumer. (Empty consumer name means @@ -231,23 +232,21 @@ public: } /** While accessing the total metrics you should have the metric lock. */ - const MetricSnapshot& getTotalMetricSnapshot(const MetricLockGuard& l) const - { + const MetricSnapshot& getTotalMetricSnapshot(const MetricLockGuard& l) const { assertMetricLockLocked(l); return *_totalMetrics; } /** While accessing snapshots you should have the metric lock. */ - const MetricSnapshot& getMetricSnapshot( - const MetricLockGuard&, - uint32_t period, bool getInProgressSet = false) const; - const MetricSnapshotSet& getMetricSnapshotSet( - const MetricLockGuard&, uint32_t period) const; - bool hasTemporarySnapshot(const MetricLockGuard& l, uint32_t period) const - { return getMetricSnapshotSet(l, period).hasTemporarySnapshot(); } + const MetricSnapshot& getMetricSnapshot( const MetricLockGuard& guard, vespalib::duration period) const { + return getMetricSnapshot(guard, period, false); + } + const MetricSnapshot& getMetricSnapshot( const MetricLockGuard&, vespalib::duration period, bool getInProgressSet) const; + const MetricSnapshotSet& getMetricSnapshotSet(const MetricLockGuard&, vespalib::duration period) const; - std::vector<uint32_t> getSnapshotPeriods(const MetricLockGuard& l) const; + std::vector<time_point::duration> getSnapshotPeriods(const MetricLockGuard& l) const; - ConsumerSpec::SP getConsumerSpec(const MetricLockGuard & guard, const Metric::String& consumer) const; + // Public only for testing. The returned pointer is only valid while holding the lock. + const ConsumerSpec * getConsumerSpec(const MetricLockGuard & guard, const Metric::String& consumer) const; /** * If you add or remove metrics from the active metric sets, normally, @@ -259,7 +258,7 @@ public: void checkMetricsAltered(const MetricLockGuard &); /** Used by unit tests to verify that we have processed for a given time. */ - time_t getLastProcessedTime() const { return _lastProcessedTime.load(std::memory_order_relaxed); } + time_point getLastProcessedTime() const { return _lastProcessedTime.load(std::memory_order_relaxed); } /** Used by unit tests to wake waiters after altering time. */ void timeChangedNotification() const; @@ -269,14 +268,14 @@ public: bool isInitialized() const; private: - void takeSnapshots(const MetricLockGuard &, time_t timeToProcess); + void takeSnapshots(const MetricLockGuard &, system_time timeToProcess); friend struct MetricManagerTest; friend struct SnapshotTest; void configure(const MetricLockGuard & guard, std::unique_ptr<MetricsmanagerConfig> conf); void run(); - time_t tick(const MetricLockGuard & guard, time_t currentTime); + time_point tick(const MetricLockGuard & guard, time_point currentTime); /** * Utility function for updating periodic metrics. * @@ -286,12 +285,12 @@ private: * without adjusting schedule for next update. * @return Time of next hook to be called in the future. */ - time_t updatePeriodicMetrics(const MetricLockGuard & guard, time_t updateTime, bool outOfSchedule); + time_point updatePeriodicMetrics(const MetricLockGuard & guard, time_point updateTime, bool outOfSchedule); void updateSnapshotMetrics(const MetricLockGuard & guard); void handleMetricsAltered(const MetricLockGuard & guard); - using SnapSpec = std::pair<uint32_t, std::string>; + using SnapSpec = std::pair<time_point::duration, std::string>; static std::vector<SnapSpec> createSnapshotPeriods( const MetricsmanagerConfig& config); void assertMetricLockLocked(const MetricLockGuard& g) const; }; diff --git a/metrics/src/vespa/metrics/metricsnapshot.cpp b/metrics/src/vespa/metrics/metricsnapshot.cpp index 1580f340f0e..6bcdcc60995 100644 --- a/metrics/src/vespa/metrics/metricsnapshot.cpp +++ b/metrics/src/vespa/metrics/metricsnapshot.cpp @@ -6,39 +6,47 @@ #include <vespa/log/log.h> LOG_SETUP(".metrics.snapshot"); +using vespalib::to_string; +using vespalib::to_s; + + namespace metrics { +static constexpr system_time system_time_epoch = system_time(); + MetricSnapshot::MetricSnapshot(const Metric::String& name) : _name(name), _period(0), - _fromTime(0), - _toTime(0), + _fromTime(system_time_epoch), + _toTime(system_time_epoch), _snapshot(new MetricSet("top", {}, "", nullptr)), _metrics() { } -MetricSnapshot::MetricSnapshot(const Metric::String& name, uint32_t period, const MetricSet& source, bool copyUnset) +MetricSnapshot::MetricSnapshot(const Metric::String& name, system_time::duration period, const MetricSet& source, bool copyUnset) : _name(name), _period(period), - _fromTime(0), - _toTime(0), + _fromTime(system_time_epoch), + _toTime(system_time_epoch), _snapshot(), _metrics() { - Metric* m = source.clone(_metrics, Metric::INACTIVE, 0, copyUnset); - assert(m->isMetricSet()); - _snapshot.reset(static_cast<MetricSet*>(m)); + _snapshot.reset(source.clone(_metrics, Metric::INACTIVE, 0, copyUnset)); _metrics.shrink_to_fit(); } MetricSnapshot::~MetricSnapshot() = default; void -MetricSnapshot::reset(time_t currentTime) +MetricSnapshot::reset() { + reset(system_time_epoch); +} +void +MetricSnapshot::reset(system_time currentTime) { _fromTime = currentTime; - _toTime = 0; + _toTime = system_time_epoch; _snapshot->reset(); } @@ -61,22 +69,19 @@ MetricSnapshot::addMemoryUsage(MemoryConsumption& mc) const { ++mc._snapshotCount; mc._snapshotName += mc.getStringMemoryUsage(_name, mc._snapshotNameUnique); - mc._snapshotMeta += sizeof(MetricSnapshot) - + _metrics.capacity() * sizeof(Metric::SP); + mc._snapshotMeta += sizeof(MetricSnapshot) + _metrics.capacity() * sizeof(Metric::SP); _snapshot->addMemoryUsage(mc); } -MetricSnapshotSet::MetricSnapshotSet( - const Metric::String& name, uint32_t period, - uint32_t count, const MetricSet& source, bool snapshotUnsetMetrics) +MetricSnapshotSet::MetricSnapshotSet(const Metric::String& name, system_time::duration period, uint32_t count, + const MetricSet& source, bool snapshotUnsetMetrics) : _count(count), _builderCount(0), - _current(new MetricSnapshot(name, period, source, snapshotUnsetMetrics)), - _building(count == 1 ? 0 : new MetricSnapshot( - name, period, source, snapshotUnsetMetrics)) + _current(std::make_unique<MetricSnapshot>(name, period, source, snapshotUnsetMetrics)), + _building(count == 1 ? nullptr : new MetricSnapshot(name, period, source, snapshotUnsetMetrics)) { - _current->reset(0); - if (_building.get()) _building->reset(0); + _current->reset(); + if (_building.get()) _building->reset(); } MetricSnapshot& @@ -87,35 +92,32 @@ MetricSnapshotSet::getNextTarget() } bool -MetricSnapshotSet::haveCompletedNewPeriod(time_t newFromTime) +MetricSnapshotSet::haveCompletedNewPeriod(system_time newFromTime) { if (_count == 1) { _current->setToTime(newFromTime); return true; } _building->setToTime(newFromTime); - // If not time to roll yet, just return + // If not time to roll yet, just return if (++_builderCount < _count) return false; - // Building buffer done. Use that as current and reset current. - MetricSnapshot::UP tmp(std::move(_current)); - _current = std::move(_building); - _building = std::move(tmp); + // Building buffer done. Use that as current and reset current. + std::swap(_current, _building); _building->reset(newFromTime); _builderCount = 0; return true; } bool -MetricSnapshotSet::timeForAnotherSnapshot(time_t currentTime) { - time_t lastTime = getToTime(); - if (currentTime >= lastTime + getPeriod()) { - if (currentTime >= lastTime + 2 * getPeriod()) { - LOG(warning, "Metric snapshot set %s was asked if it was time for " - "another snapshot, a whole period beyond when it " - "should have been done (Last update was at time %lu" - ", current time is %lu and period is %u). " - "Clearing data and updating time to current time.", - getName().c_str(), lastTime, currentTime, getPeriod()); +MetricSnapshotSet::timeForAnotherSnapshot(system_time currentTime) { + system_time lastTime = getToTime(); + vespalib::duration period = getPeriod(); + if (currentTime >= lastTime + period) { + if (currentTime >= lastTime + 2 * period) { + LOG(warning, "Metric snapshot set %s was asked if it was time for another snapshot, a whole period beyond " + "when it should have been done (Last update was at time %s, current time is %s and period " + "is %f seconds). Clearing data and updating time to current time.", + getName().c_str(), to_string(lastTime).c_str(), to_string(currentTime).c_str(), to_s(getPeriod())); reset(currentTime); } return true; @@ -124,7 +126,7 @@ MetricSnapshotSet::timeForAnotherSnapshot(time_t currentTime) { } void -MetricSnapshotSet::reset(time_t currentTime) { +MetricSnapshotSet::reset(system_time currentTime) { if (_count != 1) _building->reset(currentTime); _current->reset(currentTime); _builderCount = 0; @@ -147,7 +149,7 @@ MetricSnapshotSet::addMemoryUsage(MemoryConsumption& mc) const } void -MetricSnapshotSet::setFromTime(time_t fromTime) +MetricSnapshotSet::setFromTime(system_time fromTime) { if (_count != 1) _building->setFromTime(fromTime); _current->setFromTime(fromTime); diff --git a/metrics/src/vespa/metrics/metricsnapshot.h b/metrics/src/vespa/metrics/metricsnapshot.h index cc6ec4cbb2e..859ee4a4a97 100644 --- a/metrics/src/vespa/metrics/metricsnapshot.h +++ b/metrics/src/vespa/metrics/metricsnapshot.h @@ -15,54 +15,52 @@ namespace metrics { +using system_time = vespalib::system_time; + class MetricManager; class MetricSnapshot { Metric::String _name; // Period length of this snapshot - uint32_t _period; + system_time::duration _period; // Time this snapshot was last updated. - time_t _fromTime; + system_time _fromTime; // If set to 0, use _fromTime + _period. - time_t _toTime; + system_time _toTime; // Keeps the metrics set view of the snapshot std::unique_ptr<MetricSet> _snapshot; // Snapshots must own their own metrics mutable std::vector<Metric::UP> _metrics; public: - using UP = std::unique_ptr<MetricSnapshot>; - using SP = std::shared_ptr<MetricSnapshot>; - /** Create a fresh empty top level snapshot. */ MetricSnapshot(const Metric::String& name); /** Create a snapshot of another metric source. */ - MetricSnapshot(const Metric::String& name, uint32_t period, + MetricSnapshot(const Metric::String& name, system_time::duration period, const MetricSet& source, bool copyUnset); - virtual ~MetricSnapshot(); + ~MetricSnapshot(); - void addToSnapshot(MetricSnapshot& other, bool reset_, time_t currentTime) { + void addToSnapshot(MetricSnapshot& other, bool reset_, system_time currentTime) { _snapshot->addToSnapshot(other.getMetrics(), other._metrics); if (reset_) reset(currentTime); other._toTime = currentTime; } - void addToSnapshot(MetricSnapshot& other, time_t currentTime) const { + void addToSnapshot(MetricSnapshot& other, system_time currentTime) const { _snapshot->addToSnapshot(other.getMetrics(), other._metrics); other._toTime = currentTime; } - void setFromTime(time_t fromTime) { _fromTime = fromTime; } - void setToTime(time_t toTime) { _toTime = toTime; } + void setFromTime(system_time fromTime) { _fromTime = fromTime; } + void setToTime(system_time toTime) { _toTime = toTime; } const Metric::String& getName() const { return _name; } - uint32_t getPeriod() const { return _period; } - time_t getFromTime() const { return _fromTime; } - time_t getToTime() const { return _toTime; } - time_t getLength() const - { return (_toTime != 0 ? _toTime - _fromTime : _fromTime + _period); } + system_time::duration getPeriod() const { return _period; } + system_time getFromTime() const { return _fromTime; } + system_time getToTime() const { return _toTime; } const MetricSet& getMetrics() const { return *_snapshot; } MetricSet& getMetrics() { return *_snapshot; } - void reset(time_t currentTime); + void reset(system_time currentTime); + void reset(); /** * Recreate snapshot by cloning given metric set and then add the data * from the old one. New metrics have been added. @@ -77,37 +75,42 @@ class MetricSnapshotSet { // before we have a full time window. uint32_t _builderCount; // Number of times we've currently added to the // building instance. - MetricSnapshot::UP _current; // The last full period - MetricSnapshot::UP _building; // The building period + std::unique_ptr<MetricSnapshot> _current; // The last full period + std::unique_ptr<MetricSnapshot> _building; // The building period public: - using SP = std::shared_ptr<MetricSnapshotSet>; - - MetricSnapshotSet(const Metric::String& name, uint32_t period, - uint32_t count, const MetricSet& source, - bool snapshotUnsetMetrics); + MetricSnapshotSet(const Metric::String& name, system_time::duration period, uint32_t count, + const MetricSet& source, bool snapshotUnsetMetrics); const Metric::String& getName() const { return _current->getName(); } - uint32_t getPeriod() const { return _current->getPeriod(); } - time_t getFromTime() const { return _current->getFromTime(); } - time_t getToTime() const { return _current->getToTime(); } + system_time::duration getPeriod() const { return _current->getPeriod(); } + system_time getFromTime() const { return _current->getFromTime(); } + system_time getToTime() const { return _current->getToTime(); } + system_time getNextWorkTime() const { return getToTime() + getPeriod(); } uint32_t getCount() const { return _count; } uint32_t getBuilderCount() const { return _builderCount; } - bool hasTemporarySnapshot() const { return (_building.get() != 0); } - MetricSnapshot& getSnapshot(bool temporary = false) - { return *((temporary && _count > 1) ? _building : _current); } - const MetricSnapshot& getSnapshot(bool temporary = false) const - { return *((temporary && _count > 1) ? _building : _current); } + MetricSnapshot& getSnapshot() { + return getSnapshot(false); + } + MetricSnapshot& getSnapshot(bool temporary) { + return *((temporary && _count > 1) ? _building : _current); + } + const MetricSnapshot& getSnapshot() const { + return getSnapshot(false); + } + const MetricSnapshot& getSnapshot(bool temporary) const { + return *((temporary && _count > 1) ? _building : _current); + } MetricSnapshot& getNextTarget(); - bool timeForAnotherSnapshot(time_t currentTime); - bool haveCompletedNewPeriod(time_t newFromTime); - void reset(time_t currentTime); + bool timeForAnotherSnapshot(system_time currentTime); + bool haveCompletedNewPeriod(system_time newFromTime); + void reset(system_time currentTime); /** * Recreate snapshot by cloning given metric set and then add the data * from the old one. New metrics have been added. */ void recreateSnapshot(const MetricSet& metrics, bool copyUnset); void addMemoryUsage(MemoryConsumption&) const; - void setFromTime(time_t fromTime); + void setFromTime(system_time fromTime); }; } // metrics diff --git a/metrics/src/vespa/metrics/state_api_adapter.cpp b/metrics/src/vespa/metrics/state_api_adapter.cpp index 61a1ce7c2a9..136ccf6e06a 100644 --- a/metrics/src/vespa/metrics/state_api_adapter.cpp +++ b/metrics/src/vespa/metrics/state_api_adapter.cpp @@ -9,12 +9,12 @@ namespace metrics { vespalib::string StateApiAdapter::getMetrics(const vespalib::string &consumer) { - metrics::MetricLockGuard guard(_manager.getMetricLock()); - std::vector<uint32_t> periods = _manager.getSnapshotPeriods(guard); + MetricLockGuard guard(_manager.getMetricLock()); + auto periods = _manager.getSnapshotPeriods(guard); if (periods.empty()) { return ""; // no configuration yet } - const metrics::MetricSnapshot &snapshot(_manager.getMetricSnapshot(guard, periods[0])); + const MetricSnapshot &snapshot(_manager.getMetricSnapshot(guard, periods[0])); vespalib::asciistream json; vespalib::JsonStream stream(json); metrics::JsonWriter metricJsonWriter(stream); @@ -26,17 +26,15 @@ StateApiAdapter::getMetrics(const vespalib::string &consumer) vespalib::string StateApiAdapter::getTotalMetrics(const vespalib::string &consumer) { - _manager.updateMetrics(true); - metrics::MetricLockGuard guard(_manager.getMetricLock()); + _manager.updateMetrics(); + MetricLockGuard guard(_manager.getMetricLock()); _manager.checkMetricsAltered(guard); - time_t currentTime = vespalib::count_s(vespalib::steady_clock::now().time_since_epoch()); - auto generated = std::make_unique<metrics::MetricSnapshot>( - "Total metrics from start until current time", 0, - _manager.getTotalMetricSnapshot(guard).getMetrics(), - true); + system_time currentTime = vespalib::system_clock::now(); + auto generated = std::make_unique<MetricSnapshot>("Total metrics from start until current time", 0s, + _manager.getTotalMetricSnapshot(guard).getMetrics(), true); _manager.getActiveMetrics(guard).addToSnapshot(*generated, false, currentTime); generated->setFromTime(_manager.getTotalMetricSnapshot(guard).getFromTime()); - const metrics::MetricSnapshot &snapshot = *generated; + const MetricSnapshot &snapshot = *generated; vespalib::asciistream json; vespalib::JsonStream stream(json); metrics::JsonWriter metricJsonWriter(stream); diff --git a/metrics/src/vespa/metrics/textwriter.cpp b/metrics/src/vespa/metrics/textwriter.cpp index 94a7b7df73b..fbb31e7013a 100644 --- a/metrics/src/vespa/metrics/textwriter.cpp +++ b/metrics/src/vespa/metrics/textwriter.cpp @@ -7,9 +7,11 @@ #include "valuemetric.h" #include <sstream> +using vespalib::to_string; + namespace metrics { -TextWriter::TextWriter(std::ostream& out, uint32_t period, +TextWriter::TextWriter(std::ostream& out, vespalib::duration period, const std::string& regex, bool verbose) : _period(period), _out(out), _regex(), _verbose(verbose) { @@ -19,14 +21,14 @@ TextWriter::TextWriter(std::ostream& out, uint32_t period, } } -TextWriter::~TextWriter() { } +TextWriter::~TextWriter() = default; bool TextWriter::visitSnapshot(const MetricSnapshot& snapshot) { _out << "snapshot \"" << snapshot.getName() << "\" from " - << snapshot.getFromTime() << " to " << snapshot.getToTime() - << " period " << snapshot.getPeriod(); + << to_string(snapshot.getFromTime()) << " to " << to_string(snapshot.getToTime()) + << " period " << vespalib::count_s(snapshot.getPeriod()); return true; } @@ -82,7 +84,7 @@ bool TextWriter::visitValueMetric(const AbstractValueMetric& m, bool) { if (writeCommon(m)) { - m.print(_out, _verbose, " ", _period); + m.print(_out, _verbose, " ", vespalib::count_s(_period)); } return true; } diff --git a/metrics/src/vespa/metrics/textwriter.h b/metrics/src/vespa/metrics/textwriter.h index f060429e931..f23d1cf585c 100644 --- a/metrics/src/vespa/metrics/textwriter.h +++ b/metrics/src/vespa/metrics/textwriter.h @@ -3,20 +3,21 @@ #pragma once #include "metric.h" +#include <vespa/vespalib/util/time.h> #include <regex> #include <optional> namespace metrics { class TextWriter : public MetricVisitor { - uint32_t _period; + vespalib::duration _period; std::ostream& _out; std::vector<std::string> _path; std::optional<std::regex> _regex; bool _verbose; public: - TextWriter(std::ostream& out, uint32_t period, + TextWriter(std::ostream& out, vespalib::duration period, const std::string& regex, bool verbose); ~TextWriter(); diff --git a/metrics/src/vespa/metrics/updatehook.h b/metrics/src/vespa/metrics/updatehook.h index 9fa0d52027e..aced45b91c9 100644 --- a/metrics/src/vespa/metrics/updatehook.h +++ b/metrics/src/vespa/metrics/updatehook.h @@ -1,10 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <vespa/vespalib/util/time.h> #include <mutex> namespace metrics { +using time_point = vespalib::system_time; + class MetricLockGuard { public: MetricLockGuard(std::mutex & mutex); @@ -23,17 +26,28 @@ private: class MetricManager; class UpdateHook { - const char* _name; - time_t _nextCall; - uint32_t _period; - friend class MetricManager; - public: using MetricLockGuard = metrics::MetricLockGuard; - UpdateHook(const char* name) : _name(name), _nextCall(0), _period(0) {} + UpdateHook(const char* name, time_point::duration period) + : _name(name), + _period(period), + _nextCall() + {} virtual ~UpdateHook() = default; virtual void updateMetrics(const MetricLockGuard & guard) = 0; const char* getName() const { return _name; } + void updateNextCall() { updateNextCall(_nextCall); } + void updateNextCall(time_point now) { setNextCall(now + _period); } + bool is_periodic() const noexcept { return _period != time_point::duration::zero(); } + bool expired(time_point now) { return _nextCall <= now; } + bool has_valid_expiry() const noexcept { return _nextCall != time_point(); } + time_point::duration getPeriod() const noexcept { return _period; } + time_point getNextCall() const noexcept { return _nextCall; } + void setNextCall(time_point now) { _nextCall = now; } +private: + const char* _name; + const time_point::duration _period; + time_point _nextCall; }; } diff --git a/metrics/src/vespa/metrics/xmlwriter.cpp b/metrics/src/vespa/metrics/xmlwriter.cpp deleted file mode 100644 index 11cb450e64d..00000000000 --- a/metrics/src/vespa/metrics/xmlwriter.cpp +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "xmlwriter.h" -#include "countmetric.h" -#include "metricset.h" -#include "metricsnapshot.h" -#include "valuemetric.h" -#include <vespa/vespalib/util/xmlstream.h> -#include <sstream> - -namespace metrics { - -XmlWriter::XmlWriter(vespalib::xml::XmlOutputStream& xos, - [[maybe_unused]] uint32_t period, int verbosity) - : _xos(xos), _verbosity(verbosity) {} - -bool -XmlWriter::visitSnapshot(const MetricSnapshot& snapshot) -{ - using namespace vespalib::xml; - _xos << XmlTag("snapshot") << XmlAttribute("name", snapshot.getName()) - << XmlAttribute("from", snapshot.getFromTime()) - << XmlAttribute("to", snapshot.getToTime()) - << XmlAttribute("period", snapshot.getPeriod()); - return true; -} - -void -XmlWriter::doneVisitingSnapshot(const MetricSnapshot&) -{ - using namespace vespalib::xml; - _xos << XmlEndTag(); -} - -bool -XmlWriter::visitMetricSet(const MetricSet& set, bool) -{ - using namespace vespalib::xml; - if (set.used() || _verbosity >= 2) { - _xos << XmlTag(set.getName(), XmlTagFlags::CONVERT_ILLEGAL_CHARACTERS); - printCommonXmlParts(set); - return true; - } - return false; -} -void -XmlWriter::doneVisitingMetricSet(const MetricSet&) { - using namespace vespalib::xml; - _xos << XmlEndTag(); -} - -bool -XmlWriter::visitCountMetric(const AbstractCountMetric& metric, bool) -{ - MetricValueClass::UP values(metric.getValues()); - if (!metric.inUse(*values) && _verbosity < 2) return true; - using namespace vespalib::xml; - std::ostringstream ost; - _xos << XmlTag(metric.getName(), XmlTagFlags::CONVERT_ILLEGAL_CHARACTERS) - << XmlAttribute(metric.sumOnAdd() - ? "count" : "value", values->toString("count")); - printCommonXmlParts(metric); - _xos << XmlEndTag(); - return true; -} - -bool -XmlWriter::visitValueMetric(const AbstractValueMetric& metric, bool) -{ - MetricValueClass::UP values(metric.getValues()); - if (!metric.inUse(*values) && _verbosity < 2) return true; - using namespace vespalib::xml; - _xos << XmlTag(metric.getName(), XmlTagFlags::CONVERT_ILLEGAL_CHARACTERS) - << XmlAttribute("average", values->getLongValue("count") == 0 - ? 0 : values->getDoubleValue("total") - / values->getDoubleValue("count")) - << XmlAttribute("last", values->toString("last")); - if (!metric.summedAverage()) { - if (values->getLongValue("count") > 0) { - _xos << XmlAttribute("min", values->toString("min")) - << XmlAttribute("max", values->toString("max")); - } - _xos << XmlAttribute("count", values->getLongValue("count")); - if (_verbosity >= 2) { - _xos << XmlAttribute("total", values->toString("total")); - } - } - printCommonXmlParts(metric); - _xos << XmlEndTag(); - return true; -} - -void -XmlWriter::printCommonXmlParts(const Metric& metric) const -{ - using namespace vespalib::xml; - const Metric::Tags& tags(metric.getTags()); - if (_verbosity >= 3 && tags.size() > 0) { - std::ostringstream ost; - // XXX print tag values as well - ost << tags[0].key(); - for (uint32_t i=1; i<tags.size(); ++i) { - ost << "," << tags[i].key(); - } - _xos << XmlAttribute("tags", ost.str()); - } - if (_verbosity >= 1 && !metric.getDescription().empty()) { - _xos << XmlAttribute("description", metric.getDescription()); - } -} - -} // metrics diff --git a/metrics/src/vespa/metrics/xmlwriter.h b/metrics/src/vespa/metrics/xmlwriter.h deleted file mode 100644 index a0e8a3efeea..00000000000 --- a/metrics/src/vespa/metrics/xmlwriter.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/metrics/metric.h> -#include <vespa/vespalib/util/xmlserializable.h> - -namespace metrics { - -class XmlWriter : public MetricVisitor { - vespalib::xml::XmlOutputStream& _xos; - int _verbosity; - -public: - XmlWriter(vespalib::xml::XmlOutputStream& xos, - uint32_t period, int verbosity); - - bool visitSnapshot(const MetricSnapshot&) override; - void doneVisitingSnapshot(const MetricSnapshot&) override; - bool visitMetricSet(const MetricSet& set, bool) override; - void doneVisitingMetricSet(const MetricSet&) override; - bool visitCountMetric(const AbstractCountMetric&, bool autoGenerated) override; - bool visitValueMetric(const AbstractValueMetric&, bool autoGenerated) override; - -private: - void printCommonXmlParts(const Metric& metric) const; -}; - -} - diff --git a/model-evaluation/abi-spec.json b/model-evaluation/abi-spec.json index a5bda6e1c21..667712d0daa 100644 --- a/model-evaluation/abi-spec.json +++ b/model-evaluation/abi-spec.json @@ -47,7 +47,9 @@ }, "ai.vespa.models.evaluation.Model" : { "superClass" : "java.lang.Object", - "interfaces" : [ ], + "interfaces" : [ + "java.lang.AutoCloseable" + ], "attributes" : [ "public" ], @@ -56,7 +58,8 @@ "public java.lang.String name()", "public java.util.List functions()", "public varargs ai.vespa.models.evaluation.FunctionEvaluator evaluatorOf(java.lang.String[])", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public void close()" ], "fields" : [ ] }, @@ -67,12 +70,14 @@ "public" ], "methods" : [ + "public void <init>(com.yahoo.vespa.config.search.RankProfilesConfig, com.yahoo.vespa.config.search.core.RankingConstantsConfig, com.yahoo.vespa.config.search.core.RankingExpressionsConfig, com.yahoo.vespa.config.search.core.OnnxModelsConfig, com.yahoo.filedistribution.fileacquirer.FileAcquirer, ai.vespa.modelintegration.evaluator.OnnxRuntime)", "public void <init>(com.yahoo.vespa.config.search.RankProfilesConfig, com.yahoo.vespa.config.search.core.RankingConstantsConfig, com.yahoo.vespa.config.search.core.RankingExpressionsConfig, com.yahoo.vespa.config.search.core.OnnxModelsConfig, com.yahoo.filedistribution.fileacquirer.FileAcquirer)", "public void <init>(ai.vespa.models.evaluation.RankProfilesConfigImporter, com.yahoo.vespa.config.search.RankProfilesConfig, com.yahoo.vespa.config.search.core.RankingConstantsConfig, com.yahoo.vespa.config.search.core.RankingExpressionsConfig, com.yahoo.vespa.config.search.core.OnnxModelsConfig)", "public void <init>(java.util.Map)", "public java.util.Map models()", "public varargs ai.vespa.models.evaluation.FunctionEvaluator evaluatorOf(java.lang.String, java.lang.String[])", - "public ai.vespa.models.evaluation.Model requireModel(java.lang.String)" + "public ai.vespa.models.evaluation.Model requireModel(java.lang.String)", + "public void deconstruct()" ], "fields" : [ ] }, @@ -83,7 +88,7 @@ "public" ], "methods" : [ - "public void <init>(com.yahoo.filedistribution.fileacquirer.FileAcquirer)", + "public void <init>(com.yahoo.filedistribution.fileacquirer.FileAcquirer, ai.vespa.modelintegration.evaluator.OnnxRuntime)", "public java.util.Map importFrom(com.yahoo.vespa.config.search.RankProfilesConfig, com.yahoo.vespa.config.search.core.RankingConstantsConfig, com.yahoo.vespa.config.search.core.RankingExpressionsConfig, com.yahoo.vespa.config.search.core.OnnxModelsConfig)", "protected final java.lang.String readExpressionFromFile(java.io.File)", "protected com.yahoo.searchlib.rankingexpression.RankingExpression readExpressionFromFile(java.lang.String, com.yahoo.config.FileReference)", diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/BindingExtractor.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/BindingExtractor.java new file mode 100644 index 00000000000..6b1f60df6f4 --- /dev/null +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/BindingExtractor.java @@ -0,0 +1,183 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.models.evaluation; + +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; + +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.yahoo.searchlib.rankingexpression.Reference.RANKING_EXPRESSION_WRAPPER; + +/** + * extract information about needed bindings, arguments, and onnx models from expression functions + */ +class BindingExtractor { + + private final Map<FunctionReference, ExpressionFunction> referencedFunctions; + private final List<OnnxModel> onnxModels; + + public BindingExtractor(Map<FunctionReference, ExpressionFunction> referencedFunctions, List<OnnxModel> onnxModels) { + this.referencedFunctions = referencedFunctions; + this.onnxModels = onnxModels; + } + + static class FunctionInfo { + /** The names which may be bound externally */ + final Set<String> bindTargets = new LinkedHashSet<>(); + + /** The names which needs to be bound externally, subset of the above */ + final Set<String> arguments = new LinkedHashSet<>(); + + /** ONNX models in use */ + final Map<String, OnnxModel> onnxModelsInUse = new LinkedHashMap<>(); + + void merge(FunctionInfo other) { + bindTargets.addAll(other.bindTargets); + arguments.addAll(other.arguments); + onnxModelsInUse.putAll(other.onnxModelsInUse); + } + } + + private final Map<FunctionReference, FunctionInfo> functionsInfo = new LinkedHashMap<>(); + + FunctionInfo extractFrom(FunctionReference ref) { + if (functionsInfo.containsKey(ref)) + return functionsInfo.get(ref); + ExpressionFunction function = referencedFunctions.get(ref); + FunctionInfo result = extractFrom(function); + functionsInfo.put(ref, result); + return result; + } + + FunctionInfo extractFrom(ExpressionFunction function) { + if (function == null) + return null; + ExpressionNode functionNode = function.getBody().getRoot(); + return extractBindTargets(functionNode); + } + + private FunctionInfo extractBindTargets(ExpressionNode node) { + var result = new FunctionInfo(); + if (isFunctionReference(node)) { + var opt = FunctionReference.fromSerial(node.toString()); + if (opt.isEmpty()) { + throw new IllegalArgumentException("Could not extract function " + node + " from serialized form '" + node.toString() +"'"); + } + FunctionReference reference = opt.get(); + result.bindTargets.add(reference.serialForm()); + FunctionInfo subInfo = extractFrom(reference); + if (subInfo == null) { + // not available, must be supplied as input + result.arguments.add(reference.serialForm()); + } else { + result.merge(subInfo); + } + return result; + } + else if (isOnnx(node)) { + return extractOnnxTargets(node); + } + else if (isConstant(node)) { + result.bindTargets.add(node.toString()); + return result; + } + else if (node instanceof ReferenceNode) { + result.bindTargets.add(node.toString()); + result.arguments.add(node.toString()); + return result; + } + else if (node instanceof CompositeNode cNode) { + for (ExpressionNode child : cNode.children()) { + result.merge(extractBindTargets(child)); + } + return result; + } + if (node instanceof ConstantNode) { + return result; + } + // TODO check if more node types need consideration here + return result; + } + + /** + * Extract the feature used to evaluate the onnx model. e.g. onnx(name) and add + * that as a bind target and argument. During evaluation, this will be evaluated before + * the rest of the expression and the result is added to the context. Also extract the + * inputs to the model and add them as bind targets and arguments. + */ + private FunctionInfo extractOnnxTargets(ExpressionNode node) { + var result = new FunctionInfo(); + String onnxFeature = node.toString(); + result.bindTargets.add(onnxFeature); + Optional<String> modelName = getArgument(node); + if (modelName.isPresent()) { + for (OnnxModel onnxModel : onnxModels) { + if (onnxModel.name().equals(modelName.get())) { + // Load the model (if not already loaded) to extract inputs + onnxModel.load(); + for(String input : onnxModel.inputs().keySet()) { + result.bindTargets.add(input); + result.arguments.add(input); + } + result.onnxModelsInUse.put(onnxFeature, onnxModel); + return result; + } + } + } + // not found, must be supplied as argument + result.arguments.add(onnxFeature); + return result; + } + + private Optional<String> getArgument(ExpressionNode node) { + if (node instanceof ReferenceNode reference) { + if (reference.getArguments().size() > 0) { + var arg = reference.getArguments().expressions().get(0); + if (arg instanceof ConstantNode) { + return Optional.of(stripQuotes(arg.toString())); + } + if (arg instanceof ReferenceNode refNode) { + return Optional.of(refNode.getName()); + } + } + } + return Optional.empty(); + } + + public static String stripQuotes(String s) { + if (s.length() < 3) { + return s; + } + int lastIdx = s.length() - 1; + char first = s.charAt(0); + char last = s.charAt(lastIdx); + if (first == '"' && last == '"') return s.substring(1, lastIdx); + if (first == '\'' && last == '\'') return s.substring(1, lastIdx); + return s; + } + + private boolean isFunctionReference(ExpressionNode node) { + if ( ! (node instanceof ReferenceNode reference)) return false; + return reference.getName().equals(RANKING_EXPRESSION_WRAPPER) && reference.getArguments().size() == 1; + } + + private boolean isOnnx(ExpressionNode node) { + if ( ! (node instanceof ReferenceNode reference)) return false; + return reference.getName().equals("onnx") || reference.getName().equals("onnxModel"); + } + + private boolean isConstant(ExpressionNode node) { + if ( ! (node instanceof ReferenceNode reference)) return false; + return reference.getName().equals("constant") && reference.getArguments().size() == 1; + } + +} diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionReference.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionReference.java index 53592be7883..666c3a103b5 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionReference.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionReference.java @@ -2,6 +2,7 @@ package ai.vespa.models.evaluation; import com.yahoo.collections.Pair; +import static com.yahoo.searchlib.rankingexpression.Reference.wrapInRankingExpression; import java.util.Objects; import java.util.Optional; @@ -24,13 +25,13 @@ import java.util.regex.Pattern; class FunctionReference { private static final Pattern referencePattern = - Pattern.compile("rankingExpression\\(([a-zA-Z0-9_.]+)(@[a-f0-9]+\\.[a-f0-9]+)?\\)(\\.rankingScript)?"); + Pattern.compile("rankingExpression\\(([a-zA-Z0-9_.$]+)(@[a-f0-9]+[.a-f0-9]*)?\\)(\\.rankingScript)?"); private static final Pattern externalReferencePattern = - Pattern.compile("rankingExpression\\(([a-zA-Z0-9_.]+)(@[a-f0-9]+\\.[a-f0-9]+)?\\)(\\.expressionName)?"); + Pattern.compile("rankingExpression\\(([a-zA-Z0-9_.$]+)(@[a-f0-9]+[.a-f0-9]*)?\\)(\\.expressionName)?"); private static final Pattern argumentTypePattern = - Pattern.compile("rankingExpression\\(([a-zA-Z0-9_.]+)(@[a-f0-9]+\\.[a-f0-9]+)?\\)\\.([a-zA-Z0-9_]+)\\.type?"); + Pattern.compile("rankingExpression\\(([a-zA-Z0-9_.$]+)(@[a-f0-9]+[.a-f0-9]*)?\\)\\.([a-zA-Z0-9_]+)\\.type"); private static final Pattern returnTypePattern = - Pattern.compile("rankingExpression\\(([a-zA-Z0-9_.]+)(@[a-f0-9]+\\.[a-f0-9]+)?\\)\\.type?"); + Pattern.compile("rankingExpression\\(([a-zA-Z0-9_.$]+)(@[a-f0-9]+[.a-f0-9]*)?\\)\\.type"); /** The name of the function referenced */ private final String name; @@ -51,7 +52,8 @@ class FunctionReference { } String serialForm() { - return "rankingExpression(" + name + (instance != null ? instance : "") + ")"; + String extra = (instance != null ? instance : ""); + return wrapInRankingExpression(name + extra); } @Override diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/LazyArrayContext.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/LazyArrayContext.java index 81325740218..898f5a3a73e 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/LazyArrayContext.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/LazyArrayContext.java @@ -16,6 +16,7 @@ import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.stream.CustomCollectors; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; +import static com.yahoo.searchlib.rankingexpression.Reference.RANKING_EXPRESSION_WRAPPER; import java.util.Arrays; import java.util.HashMap; @@ -43,12 +44,12 @@ public final class LazyArrayContext extends Context implements ContextIndex { /** Create a fast lookup, lazy context for a function */ LazyArrayContext(ExpressionFunction function, + BindingExtractor bindingExtractor, Map<FunctionReference, ExpressionFunction> referencedFunctions, List<Constant> constants, - List<OnnxModel> onnxModels, Model model) { this.function = function; - this.indexedBindings = new IndexedBindings(function, referencedFunctions, constants, onnxModels, this, model); + this.indexedBindings = new IndexedBindings(function, bindingExtractor, referencedFunctions, constants, this, model); } /** @@ -185,19 +186,19 @@ public final class LazyArrayContext extends Context implements ContextIndex { * The given expression and functions may be inspected but cannot be stored. */ IndexedBindings(ExpressionFunction function, + BindingExtractor bindingExtractor, Map<FunctionReference, ExpressionFunction> referencedFunctions, List<Constant> constants, - List<OnnxModel> onnxModels, LazyArrayContext owner, - Model model) { + Model model) + { // 1. Determine and prepare bind targets - Set<String> bindTargets = new LinkedHashSet<>(); - Set<String> arguments = new LinkedHashSet<>(); // Arguments: Bind targets which need to be bound before invocation - Map<String, OnnxModel> onnxModelsInUse = new HashMap<>(); - extractBindTargets(function.getBody().getRoot(), referencedFunctions, bindTargets, arguments, onnxModels, onnxModelsInUse); + var functionInfo = bindingExtractor.extractFrom(function); + Set<String> bindTargets = functionInfo.bindTargets; + + this.onnxModels = Map.copyOf(functionInfo.onnxModelsInUse); + this.arguments = Set.copyOf(functionInfo.arguments); // Arguments: Bind targets which need to be bound before invocation - this.onnxModels = Map.copyOf(onnxModelsInUse); - this.arguments = Set.copyOf(arguments); values = new Value[bindTargets.size()]; Arrays.fill(values, missing); @@ -214,10 +215,10 @@ public final class LazyArrayContext extends Context implements ContextIndex { } } - for (Map.Entry<FunctionReference, ExpressionFunction> referencedFunction : referencedFunctions.entrySet()) { - Integer index = nameToIndex.get(referencedFunction.getKey().serialForm()); + for (FunctionReference referencedFunction : referencedFunctions.keySet()) { + Integer index = nameToIndex.get(referencedFunction.serialForm()); if (index != null) { // Referenced in this, so bind it - values[index] = new LazyValue(referencedFunction.getKey(), owner, model); + values[index] = new LazyValue(referencedFunction, owner, model); } } } @@ -226,106 +227,6 @@ public final class LazyArrayContext extends Context implements ContextIndex { missingValue = new TensorValue(value).freeze(); } - private void extractBindTargets(ExpressionNode node, - Map<FunctionReference, ExpressionFunction> functions, - Set<String> bindTargets, - Set<String> arguments, - List<OnnxModel> onnxModels, - Map<String, OnnxModel> onnxModelsInUse) { - if (isFunctionReference(node)) { - FunctionReference reference = FunctionReference.fromSerial(node.toString()).get(); - bindTargets.add(reference.serialForm()); - - ExpressionFunction function = functions.get(reference); - if (function == null) return; // Function not included in this model: Not all models are for standalone use - ExpressionNode functionNode = function.getBody().getRoot(); - extractBindTargets(functionNode, functions, bindTargets, arguments, onnxModels, onnxModelsInUse); - } - else if (isOnnx(node)) { - extractOnnxTargets(node, bindTargets, arguments, onnxModels, onnxModelsInUse); - } - else if (isConstant(node)) { - bindTargets.add(node.toString()); - } - else if (node instanceof ReferenceNode) { - bindTargets.add(node.toString()); - arguments.add(node.toString()); - } - else if (node instanceof CompositeNode cNode) { - for (ExpressionNode child : cNode.children()) - extractBindTargets(child, functions, bindTargets, arguments, onnxModels, onnxModelsInUse); - } - } - - /** - * Extract the feature used to evaluate the onnx model. e.g. onnx(name) and add - * that as a bind target and argument. During evaluation, this will be evaluated before - * the rest of the expression and the result is added to the context. Also extract the - * inputs to the model and add them as bind targets and arguments. - */ - private void extractOnnxTargets(ExpressionNode node, - Set<String> bindTargets, - Set<String> arguments, - List<OnnxModel> onnxModels, - Map<String, OnnxModel> onnxModelsInUse) { - Optional<String> modelName = getArgument(node); - if (modelName.isPresent()) { - for (OnnxModel onnxModel : onnxModels) { - if (onnxModel.name().equals(modelName.get())) { - String onnxFeature = node.toString(); - bindTargets.add(onnxFeature); - - // Load the model (if not already loaded) to extract inputs - onnxModel.load(); - - for(String input : onnxModel.inputs().keySet()) { - bindTargets.add(input); - arguments.add(input); - } - onnxModelsInUse.put(onnxFeature, onnxModel); - } - } - } - } - - private Optional<String> getArgument(ExpressionNode node) { - if (node instanceof ReferenceNode reference) { - if (reference.getArguments().size() > 0) { - if (reference.getArguments().expressions().get(0) instanceof ConstantNode) { - ExpressionNode constantNode = reference.getArguments().expressions().get(0); - return Optional.of(stripQuotes(constantNode.toString())); - } - if (reference.getArguments().expressions().get(0) instanceof ReferenceNode refNode) { - return Optional.of(refNode.getName()); - } - } - } - return Optional.empty(); - } - - public static String stripQuotes(String s) { - if (s.codePointAt(0) == '"' && s.codePointAt(s.length()-1) == '"') - return s.substring(1, s.length()-1); - if (s.codePointAt(0) == '\'' && s.codePointAt(s.length()-1) == '\'') - return s.substring(1, s.length()-1); - return s; - } - - private boolean isFunctionReference(ExpressionNode node) { - if ( ! (node instanceof ReferenceNode reference)) return false; - return reference.getName().equals("rankingExpression") && reference.getArguments().size() == 1; - } - - private boolean isOnnx(ExpressionNode node) { - if ( ! (node instanceof ReferenceNode reference)) return false; - return reference.getName().equals("onnx") || reference.getName().equals("onnxModel"); - } - - private boolean isConstant(ExpressionNode node) { - if ( ! (node instanceof ReferenceNode reference)) return false; - return reference.getName().equals("constant") && reference.getArguments().size() == 1; - } - Value get(int index) { Value value = values[index]; return value == missing ? missingValue : value; diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/Model.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/Model.java index d66d0330ea6..f173a6b453f 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/Model.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/Model.java @@ -10,7 +10,6 @@ import com.yahoo.tensor.TensorType; import java.util.Arrays; import java.util.Collection; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -22,7 +21,7 @@ import java.util.stream.Collectors; * @author bratseth */ @Beta -public class Model { +public class Model implements AutoCloseable { /** The prefix generated by model-integration/../IntermediateOperation */ private final static String INTERMEDIATE_OPERATION_FUNCTION_PREFIX = "imported_ml_function_"; @@ -43,11 +42,14 @@ public class Model { private final ExpressionOptimizer expressionOptimizer = new ExpressionOptimizer(); + private final List<Runnable> closeActions; + /** Programmatically create a model containing functions without constant of function references only */ public Model(String name, Collection<ExpressionFunction> functions) { this(name, functions.stream().collect(Collectors.toMap(f -> FunctionReference.fromName(f.getName()), f -> f)), Map.of(), + Map.of(), List.of(), List.of()); } @@ -55,15 +57,18 @@ public class Model { Model(String name, Map<FunctionReference, ExpressionFunction> functions, Map<FunctionReference, ExpressionFunction> referencedFunctions, + Map<String, TensorType> declaredTypes, List<Constant> constants, List<OnnxModel> onnxModels) { this.name = name; + var bindingExtractor = new BindingExtractor(referencedFunctions, onnxModels); + // Build context and add missing function arguments (missing because it is legal to omit scalar type arguments) Map<String, LazyArrayContext> contextBuilder = new LinkedHashMap<>(); for (Map.Entry<FunctionReference, ExpressionFunction> function : functions.entrySet()) { try { - LazyArrayContext context = new LazyArrayContext(function.getValue(), referencedFunctions, constants, onnxModels, this); + LazyArrayContext context = new LazyArrayContext(function.getValue(), bindingExtractor, referencedFunctions, constants, this); contextBuilder.put(function.getValue().getName(), context); if (function.getValue().returnType().isEmpty()) { functions.put(function.getKey(), function.getValue().withReturnType(TensorType.empty)); @@ -84,8 +89,10 @@ public class Model { } else { // External functions have type info (when not scalar) - add argument types - if (function.getValue().getArgumentType(argument) == null) - functions.put(function.getKey(), function.getValue().withArgument(argument, TensorType.empty)); + if (function.getValue().getArgumentType(argument) == null) { + TensorType type = declaredTypes.getOrDefault(argument, TensorType.empty); + functions.put(function.getKey(), function.getValue().withArgument(argument, type)); + } } } } @@ -94,13 +101,18 @@ public class Model { } } this.contextPrototypes = Map.copyOf(contextBuilder); - this.functions = List.copyOf(functions.values()); + // Optimize free functions + this.functions = List.copyOf(functions.entrySet() + .stream() + .map(f -> optimize(f.getValue(), + contextPrototypes.get(f.getKey().functionName()))) + .collect(Collectors.toList())); + this.publicFunctions = functions.values().stream() .filter(f -> !f.getName().startsWith(INTERMEDIATE_OPERATION_FUNCTION_PREFIX)).toList(); - // Optimize functions - this.referencedFunctions = Map.copyOf(referencedFunctions.entrySet().stream() - .collect(CustomCollectors.toLinkedMap(f -> f.getKey(), f -> optimize(f.getValue(), contextPrototypes.get(f.getKey().functionName()))))); + this.referencedFunctions = Map.copyOf(referencedFunctions); + this.closeActions = onnxModels.stream().map(o -> (Runnable)o::close).toList(); } /** Returns an optimized version of the given function */ @@ -223,4 +235,5 @@ public class Model { @Override public String toString() { return "model '" + name + "'"; } + @Override public void close() { closeActions.forEach(Runnable::run); } } diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java index 28b613ca281..fd5306f9add 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.evaluation; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import com.yahoo.api.annotations.Beta; import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; @@ -30,8 +31,17 @@ public class ModelsEvaluator extends AbstractComponent { RankingConstantsConfig constantsConfig, RankingExpressionsConfig expressionsConfig, OnnxModelsConfig onnxModelsConfig, + FileAcquirer fileAcquirer, + OnnxRuntime onnx) { + this(new RankProfilesConfigImporter(fileAcquirer, onnx), config, constantsConfig, expressionsConfig, onnxModelsConfig); + } + + public ModelsEvaluator(RankProfilesConfig config, + RankingConstantsConfig constantsConfig, + RankingExpressionsConfig expressionsConfig, + OnnxModelsConfig onnxModelsConfig, FileAcquirer fileAcquirer) { - this(new RankProfilesConfigImporter(fileAcquirer), config, constantsConfig, expressionsConfig, onnxModelsConfig); + this(config, constantsConfig, expressionsConfig, onnxModelsConfig, fileAcquirer, new OnnxRuntime()); } public ModelsEvaluator(RankProfilesConfigImporter importer, @@ -69,4 +79,5 @@ public class ModelsEvaluator extends AbstractComponent { return model; } + @Override public void deconstruct() { models.values().forEach(Model::close); } } diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/OnnxExpressionNode.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/OnnxExpressionNode.java new file mode 100644 index 00000000000..a50d9e36d74 --- /dev/null +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/OnnxExpressionNode.java @@ -0,0 +1,102 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.models.evaluation; + +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.searchlib.rankingexpression.evaluation.Context; +import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.searchlib.rankingexpression.evaluation.Value; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.evaluation.TypeContext; + +import java.util.ArrayList; +import java.util.Deque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * make it possible to evaluate an ONNX model anywhere in the ranking expression tree + */ +class OnnxExpressionNode extends CompositeNode { + private final OnnxModel model; + private final String onnxOutputName; + private final TensorType expectedType; + private final String outputAs; + private final List<String> modelInputs = new ArrayList<>(); + private final List<ExpressionNode> inputRefs = new ArrayList<>(); + + OnnxExpressionNode(OnnxModel model, String onnxOutputName, TensorType expectedType, String outputAs) { + this.model = model; + this.onnxOutputName = onnxOutputName; + this.expectedType = expectedType; + this.outputAs = outputAs; + for (var input : model.inputSpecs) { + modelInputs.add(input.onnxName); + var optRef = parseOnnxInput(input.source); + if (optRef.isEmpty()) { + throw new IllegalArgumentException("Bad input source for ONNX model " + model.name() + ": '" + input + "'"); + } + var ref = optRef.get(); + inputRefs.add(new ReferenceNode(ref)); + } + } + + static Optional<Reference> parseOnnxInput(String input) { + var optRef = Reference.simple(input); + if (optRef.isPresent()) { + return optRef; + } + try { + var ref = Reference.fromIdentifier(input); + return Optional.of(ref); + } catch (Exception e) { + // fallthrough + } + return Optional.empty(); + } + + @Override + public List<ExpressionNode> children() { return List.copyOf(inputRefs); } + + @Override + public CompositeNode setChildren(List<ExpressionNode> children) { + if (inputRefs.size() != children.size()) { + throw new IllegalArgumentException("bad setChildren"); + } + inputRefs.clear(); + inputRefs.addAll(children); + return this; + } + + @Override + public Value evaluate(Context context) { + Map<String, Tensor> inputs = new HashMap<>(); + for (int i = 0; i < modelInputs.size(); i++) { + Value inputValue = inputRefs.get(i).evaluate(context); + inputs.put(modelInputs.get(i), inputValue.asTensor()); + } + return new TensorValue(model.evaluate(inputs, onnxOutputName)); + } + + @Override + public TensorType type(TypeContext<Reference> context) { return expectedType; } + + @Override + public int hashCode() { return Objects.hash("OnnxExpressionNode", model.name(), onnxOutputName); } + + @Override + public StringBuilder toString(StringBuilder b, SerializationContext context, Deque<String> path, CompositeNode parent) { + b.append("onnx_expression_node(").append(model.name()).append(")"); + if (outputAs != null && ! outputAs.equals("")) { + b.append(".").append(outputAs); + } + return b; + } +} diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/OnnxModel.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/OnnxModel.java index 19a9a1dccd5..59febf7cdbf 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/OnnxModel.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/OnnxModel.java @@ -3,10 +3,17 @@ package ai.vespa.models.evaluation; import ai.vespa.modelintegration.evaluator.OnnxEvaluator; import ai.vespa.modelintegration.evaluator.OnnxEvaluatorOptions; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; /** @@ -14,18 +21,59 @@ import java.util.Map; * * @author lesters */ -class OnnxModel { +class OnnxModel implements AutoCloseable { + + static class InputSpec { + String onnxName; + String source; + TensorType wantedType; + InputSpec(String name, String source, TensorType tType) { + this.onnxName = name; + this.source = source; + this.wantedType = tType; + } + InputSpec(String name, String source) { this(name, source, null); } + } + + static class OutputSpec { + String onnxName; + String outputAs; + TensorType expectedType; + OutputSpec(String name, String as, TensorType tType) { + this.onnxName = name; + this.outputAs = as; + this.expectedType = tType; + } + OutputSpec(String name, String as) { this(name, as, null); } + } + + final List<InputSpec> inputSpecs = new ArrayList<>(); + final List<OutputSpec> outputSpecs = new ArrayList<>(); + + void addInputMapping(String onnxName, String source) { + if (evaluator != null) + throw new IllegalStateException("input mapping must be added before load()"); + inputSpecs.add(new InputSpec(onnxName, source)); + } + void addOutputMapping(String onnxName, String outputAs) { + if (evaluator != null) + throw new IllegalStateException("output mapping must be added before load()"); + outputSpecs.add(new OutputSpec(onnxName, outputAs)); + } private final String name; private final File modelFile; private final OnnxEvaluatorOptions options; + private final OnnxRuntime onnx; private OnnxEvaluator evaluator; + private final Map<String, ExpressionNode> exprPerOutput = new HashMap<>(); - OnnxModel(String name, File modelFile, OnnxEvaluatorOptions options) { + OnnxModel(String name, File modelFile, OnnxEvaluatorOptions options, OnnxRuntime onnx) { this.name = name; this.modelFile = modelFile; this.options = options; + this.onnx = onnx; } public String name() { @@ -34,20 +82,118 @@ class OnnxModel { public void load() { if (evaluator == null) { - evaluator = new OnnxEvaluator(modelFile.getPath(), options); + evaluator = onnx.evaluatorOf(modelFile.getPath(), options); + fillInputTypes(evaluator().getInputs()); + fillOutputTypes(evaluator().getOutputs()); + fillOutputExpressions(); + } + } + + void fillInputTypes(Map<String, OnnxEvaluator.IdAndType> wantedTypes) { + if (inputSpecs.isEmpty()) { + for (var entry : wantedTypes.entrySet()) { + String name = entry.getKey(); + String source = entry.getValue().id(); + TensorType tType = entry.getValue().type(); + var spec = new InputSpec(name, source, tType); + inputSpecs.add(spec); + } + } else { + if (wantedTypes.size() != inputSpecs.size()) { + throw new IllegalArgumentException("Onnx model " + name() + + ": Mismatch between " + inputSpecs.size() + + " configured inputs and " + + wantedTypes.size() + " actual model inputs"); + } + for (var spec : inputSpecs) { + var entry = wantedTypes.get(spec.onnxName); + if (entry == null) { + throw new IllegalArgumentException("Onnx model " + name() + + ": No type in actual model for configured input " + + spec.onnxName); + } + spec.wantedType = entry.type(); + } + } + } + + void fillOutputTypes(Map<String, OnnxEvaluator.IdAndType> outputTypes) { + if (outputSpecs.isEmpty()) { + for (var entry : outputTypes.entrySet()) { + String name = entry.getKey(); + String as = entry.getValue().id(); + TensorType tType = entry.getValue().type(); + var spec = new OutputSpec(name, as, tType); + outputSpecs.add(spec); + } + } else { + if (outputTypes.size() != outputSpecs.size()) { + throw new IllegalArgumentException("Onnx model " + name() + + ": Mismatch between " + outputSpecs.size() + + " configured outputs and " + + outputTypes.size() + " actual model outputs"); + } + for (var spec : outputSpecs) { + var entry = outputTypes.get(spec.onnxName); + if (entry == null) { + throw new IllegalArgumentException("Onnx model " + name() + + ": No type in actual model for configured output " + + spec.onnxName); + } + spec.expectedType = entry.type(); + } } } public Map<String, TensorType> inputs() { - return evaluator().getInputInfo(); + var map = new HashMap<String, TensorType>(); + for (var spec : inputSpecs) { + map.put(spec.source, spec.wantedType); + } + return map; } public Map<String, TensorType> outputs() { - return evaluator().getOutputInfo(); + var map = new HashMap<String, TensorType>(); + for (var spec : outputSpecs) { + map.put(spec.outputAs, spec.expectedType); + } + return map; + } + + void fillOutputExpressions() { + for (var spec : outputSpecs) { + var node = new OnnxExpressionNode(this, spec.onnxName, spec.expectedType, spec.outputAs); + exprPerOutput.put(spec.outputAs, node); + } + } + + ExpressionNode getExpressionForOutput(String outputName) { + if (outputName == null && exprPerOutput.size() == 1) { + return exprPerOutput.values().iterator().next(); + } + return exprPerOutput.get(outputName); } public Tensor evaluate(Map<String, Tensor> inputs, String output) { - return evaluator().evaluate(inputs, output); + var mapped = new HashMap<String, Tensor>(); + for (var spec : inputSpecs) { + Tensor val = inputs.get(spec.source); + if (val == null) { + throw new IllegalArgumentException("evaluate ONNX model " + name() + ": missing input from source " + spec.source); + } + mapped.put(spec.onnxName, val); + } + String onnxName = null; + for (var spec : outputSpecs) { + if (spec.outputAs.equals(output)) { + onnxName = spec.onnxName; + } + } + if (onnxName == null) { + throw new IllegalArgumentException("evaluate ONNX model " + name() + ": no output available as: " + output); + } + return evaluator().evaluate(mapped, onnxName); } private OnnxEvaluator evaluator() { @@ -57,4 +203,5 @@ class OnnxModel { return evaluator; } + @Override public void close() { evaluator.close(); } } diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java index e8aae24ca9e..76869932a3e 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java @@ -2,6 +2,7 @@ package ai.vespa.models.evaluation; import ai.vespa.modelintegration.evaluator.OnnxEvaluatorOptions; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import com.yahoo.collections.Pair; import com.yahoo.config.FileReference; import com.yahoo.filedistribution.fileacquirer.FileAcquirer; @@ -46,9 +47,11 @@ import java.util.regex.Pattern; public class RankProfilesConfigImporter { private final FileAcquirer fileAcquirer; + private final OnnxRuntime onnx; - public RankProfilesConfigImporter(FileAcquirer fileAcquirer) { + public RankProfilesConfigImporter(FileAcquirer fileAcquirer, OnnxRuntime onnx) { this.fileAcquirer = fileAcquirer; + this.onnx = onnx; } /** @@ -87,11 +90,14 @@ public class RankProfilesConfigImporter { SmallConstantsInfo smallConstantsInfo = new SmallConstantsInfo(); ExpressionFunction firstPhase = null; ExpressionFunction secondPhase = null; + ExpressionFunction globalPhase = null; + Map<String, TensorType> declaredTypes = new LinkedHashMap<>(); for (RankProfilesConfig.Rankprofile.Fef.Property property : profile.fef().property()) { Optional<FunctionReference> reference = FunctionReference.fromSerial(property.name()); Optional<FunctionReference> externalReference = FunctionReference.fromExternalSerial(property.name()); Optional<Pair<FunctionReference, String>> argumentType = FunctionReference.fromTypeArgumentSerial(property.name()); Optional<FunctionReference> returnType = FunctionReference.fromReturnTypeSerial(property.name()); + Optional<String> typeDeclaredFeature = fromTypeDeclarationSerial(property.name()); if (externalReference.isPresent()) { RankingExpression expression = largeExpressions.get(property.value()); ExpressionFunction function = new ExpressionFunction(externalReference.get().functionName(), @@ -123,11 +129,15 @@ public class RankProfilesConfigImporter { referencedFunctions.put(argReference, function); } else if (returnType.isPresent()) { // Return type always follows the function in properties - ExpressionFunction function = referencedFunctions.get(returnType.get()); - function = function.withReturnType(TensorType.fromSpec(property.value())); - if (returnType.get().isFree()) - functions.put(returnType.get(), function); - referencedFunctions.put(returnType.get(), function); + FunctionReference functionRef = returnType.get(); + ExpressionFunction function = referencedFunctions.get(functionRef); + TensorType type = TensorType.fromSpec(property.value()); + function = function.withReturnType(type); + if (functionRef.isFree()) + functions.put(functionRef, function); + referencedFunctions.put(functionRef, function); + declaredTypes.put(function.getName(), type); // "foo" + declaredTypes.put(functionRef.serialForm(), type); // "rankingExpression(foo)" } else if (property.name().equals("vespa.rank.firstphase")) { // Include in addition to functions firstPhase = new ExpressionFunction("firstphase", new ArrayList<>(), @@ -137,6 +147,13 @@ public class RankProfilesConfigImporter { secondPhase = new ExpressionFunction("secondphase", new ArrayList<>(), new RankingExpression("second-phase", property.value())); } + else if (property.name().equals("vespa.rank.globalphase")) { // Include in addition to functions + globalPhase = new ExpressionFunction("globalphase", new ArrayList<>(), + new RankingExpression("global-phase", property.value())); + } + else if (typeDeclaredFeature.isPresent()) { + declaredTypes.put(typeDeclaredFeature.get(), TensorType.fromSpec(property.value())); + } else { smallConstantsInfo.addIfSmallConstantInfo(property.name(), property.value()); } @@ -145,11 +162,13 @@ public class RankProfilesConfigImporter { functions.put(FunctionReference.fromName("firstphase"), firstPhase); if (functionByName("secondphase", functions.values()) == null && secondPhase != null) // may be already included, depending on body functions.put(FunctionReference.fromName("secondphase"), secondPhase); + if (functionByName("globalphase", functions.values()) == null && globalPhase != null) // may be already included, depending on body + functions.put(FunctionReference.fromName("globalphase"), globalPhase); constants.addAll(smallConstantsInfo.asConstants()); try { - return new Model(profile.name(), functions, referencedFunctions, constants, onnxModels); + return new Model(profile.name(), functions, referencedFunctions, declaredTypes, constants, onnxModels); } catch (RuntimeException e) { throw new IllegalArgumentException("Could not load model '" + profile.name() + "'", e); @@ -183,7 +202,14 @@ public class RankProfilesConfigImporter { options.setInterOpThreads(onnxModelConfig.stateless_interop_threads()); options.setIntraOpThreads(onnxModelConfig.stateless_intraop_threads()); options.setGpuDevice(onnxModelConfig.gpu_device(), onnxModelConfig.gpu_device_required()); - return new OnnxModel(name, file, options); + var m = new OnnxModel(name, file, options, onnx); + for (var spec : onnxModelConfig.input()) { + m.addInputMapping(spec.name(), spec.source()); + } + for (var spec : onnxModelConfig.output()) { + m.addOutputMapping(spec.name(), spec.as()); + } + return m; } catch (InterruptedException e) { throw new IllegalStateException("Gave up waiting for ONNX model " + onnxModelConfig.name()); } @@ -289,4 +315,15 @@ public class RankProfilesConfigImporter { } + private static final Pattern typeDeclarationPattern = + Pattern.compile("vespa[.]type[.]([a-zA-Z0-9]+)[.](.+)"); + + static Optional<String> fromTypeDeclarationSerial(String serialForm) { + Matcher expressionMatcher = typeDeclarationPattern.matcher(serialForm); + if ( ! expressionMatcher.matches()) return Optional.empty(); + String name = expressionMatcher.group(1); + String argument = expressionMatcher.group(2); + return Optional.of(name + "(" + argument + ")"); + } + } diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java index ab2f53db863..3f9b86e67e4 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.evaluation; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; import com.yahoo.path.Path; @@ -9,6 +10,9 @@ import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; + +import java.io.File; +import java.util.HashMap; import java.util.Map; import static org.junit.Assert.assertEquals; @@ -36,8 +40,19 @@ public class ModelTester { RankingExpressionsConfig expressionsConfig = ConfigGetter.getConfig(RankingExpressionsConfig.class, fileConfigId(path, "ranking-expressions.cfg")); OnnxModelsConfig onnxModelsConfig = ConfigGetter.getConfig(OnnxModelsConfig.class, fileConfigId(path, "onnx-models.cfg")); + Map<String, File> fileMap = new HashMap<>(); + for (var cfgEntry : onnxModelsConfig.model()) { + fileMap.put(cfgEntry.fileref().value(), new File(path + cfgEntry.fileref().value())); + } + for (var cfgEntry : constantsConfig.constant()) { + fileMap.put(cfgEntry.fileref().value(), new File(path + cfgEntry.fileref().value())); + } + for (var cfgEntry : expressionsConfig.expression()) { + fileMap.put(cfgEntry.fileref().value(), new File(path + cfgEntry.fileref().value())); + } + var fileAcquirer = MockFileAcquirer.returnFiles(fileMap); - return new RankProfilesConfigImporterWithMockedConstants(Path.fromString(path).append("constants"), MockFileAcquirer.returnFile(null)) + return new RankProfilesConfigImporter(fileAcquirer, new OnnxRuntime()) .importFrom(config, constantsConfig, expressionsConfig, onnxModelsConfig); } diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java index 3cd04db8edd..126d3492039 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java @@ -142,7 +142,7 @@ public class ModelsEvaluatorTest { RankingExpressionsConfig expressionsConfig = ConfigGetter.getConfig(RankingExpressionsConfig.class, fileConfigId("ranking-expressions.cfg")); OnnxModelsConfig onnxModelsConfig = ConfigGetter.getConfig(OnnxModelsConfig.class, fileConfigId("onnx-models.cfg")); - return new ModelsEvaluator(new RankProfilesConfigImporterWithMockedConstants(Path.fromString(CONFIG_DIR).append("constants"), MockFileAcquirer.returnFile(null)), + return new ModelsEvaluator(new RankProfilesConfigImporterWithMockedConstants(Path.fromString(CONFIG_DIR), MockFileAcquirer.returnFile(null)), config, constantsConfig, expressionsConfig, onnxModelsConfig); } diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java index 992dae22aaf..0bee33be3cc 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.evaluation; -import ai.vespa.modelintegration.evaluator.OnnxEvaluator; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; @@ -30,7 +30,7 @@ public class OnnxEvaluatorTest { @Test public void testOnnxEvaluation() { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); ModelsEvaluator models = createModels(); assertTrue(models.models().containsKey("add_mul")); diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfileImportingTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfileImportingTest.java index 3fdbb370a5c..65eb55ae46d 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfileImportingTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfileImportingTest.java @@ -31,4 +31,33 @@ public class RankProfileImportingTest { "4 * (match + rankBoost)", macros); } + @Test + public void testImportingSimpleGlobalPhase() { + ModelTester tester = new ModelTester("src/test/resources/config/dotproduct/"); + assertEquals(1, tester.models().size()); + Model m = tester.models().get("default"); + assertEquals("default", m.name()); + assertEquals(1, m.functions().size()); + tester.assertFunction("globalphase", "reduce(attribute(aa) * query(zz), sum)", m); + var f = m.functions().get(0); + assertEquals("globalphase", f.getName()); + assertEquals(2, f.arguments().size()); + assertEquals("tensor(d0[3])", f.getArgumentType("query(zz)").toString()); + assertEquals("tensor(d0[3])", f.getArgumentType("attribute(aa)").toString()); + var rt = f.returnType(); + assertEquals(true, rt.isPresent()); + assertEquals("tensor()", rt.get().toString()); + } + + @Test + public void testImportingExpressionsAsArguments() { + ModelTester tester = new ModelTester("src/test/resources/config/expressions-as-arguments/"); + assertEquals(3, tester.models().size()); + } + + @Test + public void testImportingWithMacros() { + ModelTester tester = new ModelTester("src/test/resources/config/ranking-macros/"); + assertEquals(5, tester.models().size()); + } } diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java index c11f4764678..c166128549f 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.evaluation; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import com.yahoo.config.FileReference; import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.io.GrowableByteBuffer; @@ -24,15 +25,17 @@ public class RankProfilesConfigImporterWithMockedConstants extends RankProfilesC private final Path constantsPath; public RankProfilesConfigImporterWithMockedConstants(Path constantsPath, FileAcquirer fileAcquirer) { - super(fileAcquirer); + super(fileAcquirer, new OnnxRuntime()); this.constantsPath = constantsPath; } @Override protected Tensor readTensorFromFile(String name, TensorType type, FileReference fileReference) { try { + var path = constantsPath.append(fileReference.value()); + var file = path.toFile(); return TypedBinaryFormat.decode(Optional.of(type), - GrowableByteBuffer.wrap(IOUtils.readFileBytes(constantsPath.append(name).toFile()))); + GrowableByteBuffer.wrap(IOUtils.readFileBytes(file))); } catch (IOException e) { log.warning("Missing a mocked tensor constant for '" + name + "': " + e.getMessage() + diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/TinyBertTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/TinyBertTest.java new file mode 100644 index 00000000000..dd240454928 --- /dev/null +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/TinyBertTest.java @@ -0,0 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.models.evaluation; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author arnej + */ +public class TinyBertTest { + + @Test + public void testTinyBert() { + ModelTester tester = new ModelTester("src/test/resources/config/tinybert/"); + assertEquals(3, tester.models().size()); + } + +} diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/HandlerTester.java b/model-evaluation/src/test/java/ai/vespa/models/handler/HandlerTester.java index 6c4dd886f4b..38215858366 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/HandlerTester.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/HandlerTester.java @@ -27,16 +27,22 @@ class HandlerTester { } private static Predicate<String> matchString(String expected) { return s -> { - //System.out.println("Expected: " + expected); - //System.out.println("Actual: " + s); - return expected.equals(s); + boolean result = expected.equals(s); + if (!result) { + System.out.println("Expected: " + expected); + System.out.println("Actual: " + s); + } + return result; }; } private static Predicate<String> matchJsonString(String expected) { return s -> { - //System.out.println("Expected: " + expected); - //System.out.println("Actual: " + s); - return JSON.canonical(expected).equals(JSON.canonical(s)); + boolean result = JSON.canonical(expected).equals(JSON.canonical(s)); + if (!result) { + System.out.println("Expected: " + expected); + System.out.println("Actual: " + s); + } + return result; }; } public static Predicate<String> matchJson(String... expectedJson) { diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java index 9b2b793212b..8b6cad1914f 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.handler; -import ai.vespa.modelintegration.evaluator.OnnxEvaluator; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import ai.vespa.models.evaluation.ModelsEvaluator; import ai.vespa.models.evaluation.RankProfilesConfigImporterWithMockedConstants; import com.yahoo.config.subscription.ConfigGetter; @@ -323,7 +323,7 @@ public class ModelsEvaluationHandlerTest { @Test public void testMnistSavedEvaluateSpecificFunction() { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); Map<String, String> properties = new HashMap<>(); properties.put("input", inputTensor()); properties.put("format.tensors", "long"); @@ -349,7 +349,7 @@ public class ModelsEvaluationHandlerTest { RankingExpressionsConfig expressionsConfig = ConfigGetter.getConfig(RankingExpressionsConfig.class, fileConfigId("ranking-expressions.cfg")); OnnxModelsConfig onnxModelsConfig = ConfigGetter.getConfig(OnnxModelsConfig.class, fileConfigId("onnx-models.cfg")); - return new ModelsEvaluator(new RankProfilesConfigImporterWithMockedConstants(Path.fromString(MODELS_DIR).append("constants"), MockFileAcquirer.returnFile(null)), + return new ModelsEvaluator(new RankProfilesConfigImporterWithMockedConstants(Path.fromString(MODELS_DIR), MockFileAcquirer.returnFile(null)), config, constantsConfig, expressionsConfig, onnxModelsConfig); } diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java index 86f56e14e2d..856031da72f 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.handler; -import ai.vespa.modelintegration.evaluator.OnnxEvaluator; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import ai.vespa.models.evaluation.ModelsEvaluator; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.filedistribution.fileacquirer.FileAcquirer; @@ -27,7 +27,7 @@ public class OnnxEvaluationHandlerTest { @BeforeClass static public void setUp() { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); handler = new HandlerTester(createModels()); } diff --git a/model-evaluation/src/test/resources/config/dotproduct/onnx-models.cfg b/model-evaluation/src/test/resources/config/dotproduct/onnx-models.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/model-evaluation/src/test/resources/config/dotproduct/onnx-models.cfg diff --git a/model-evaluation/src/test/resources/config/dotproduct/rank-profiles.cfg b/model-evaluation/src/test/resources/config/dotproduct/rank-profiles.cfg new file mode 100644 index 00000000000..ae1e6791f3e --- /dev/null +++ b/model-evaluation/src/test/resources/config/dotproduct/rank-profiles.cfg @@ -0,0 +1,9 @@ +rankprofile[0].name "default" +rankprofile[0].fef.property[0].name "vespa.rank.globalphase" +rankprofile[0].fef.property[0].value "sum(attribute(aa) * query(zz))" +rankprofile[0].fef.property[1].name "vespa.match.feature" +rankprofile[0].fef.property[1].value "attribute(aa)" +rankprofile[0].fef.property[2].name "vespa.type.attribute.aa" +rankprofile[0].fef.property[2].value "tensor(d0[3])" +rankprofile[0].fef.property[3].name "vespa.type.query.zz" +rankprofile[0].fef.property[3].value "tensor(d0[3])" diff --git a/model-evaluation/src/test/resources/config/dotproduct/ranking-constants.cfg b/model-evaluation/src/test/resources/config/dotproduct/ranking-constants.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/model-evaluation/src/test/resources/config/dotproduct/ranking-constants.cfg diff --git a/model-evaluation/src/test/resources/config/dotproduct/ranking-expressions.cfg b/model-evaluation/src/test/resources/config/dotproduct/ranking-expressions.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/model-evaluation/src/test/resources/config/dotproduct/ranking-expressions.cfg diff --git a/model-evaluation/src/test/resources/config/expressions-as-arguments/onnx-models.cfg b/model-evaluation/src/test/resources/config/expressions-as-arguments/onnx-models.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/model-evaluation/src/test/resources/config/expressions-as-arguments/onnx-models.cfg diff --git a/model-evaluation/src/test/resources/config/expressions-as-arguments/rank-profiles.cfg b/model-evaluation/src/test/resources/config/expressions-as-arguments/rank-profiles.cfg new file mode 100644 index 00000000000..d8ccbeaf0fb --- /dev/null +++ b/model-evaluation/src/test/resources/config/expressions-as-arguments/rank-profiles.cfg @@ -0,0 +1,175 @@ +rankprofile[0].name "default" +rankprofile[0].fef.property[0].name "vespa.type.attribute.t1" +rankprofile[0].fef.property[0].value "tensor<float>(x{})" +rankprofile[0].fef.property[1].name "vespa.type.attribute.t2" +rankprofile[0].fef.property[1].value "tensor<float>(x{})" +rankprofile[1].name "unranked" +rankprofile[1].fef.property[0].name "vespa.rank.firstphase" +rankprofile[1].fef.property[0].value "value(0)" +rankprofile[1].fef.property[1].name "vespa.hitcollector.heapsize" +rankprofile[1].fef.property[1].value "0" +rankprofile[1].fef.property[2].name "vespa.hitcollector.arraysize" +rankprofile[1].fef.property[2].value "0" +rankprofile[1].fef.property[3].name "vespa.dump.ignoredefaultfeatures" +rankprofile[1].fef.property[3].value "true" +rankprofile[1].fef.property[4].name "vespa.type.attribute.t1" +rankprofile[1].fef.property[4].value "tensor<float>(x{})" +rankprofile[1].fef.property[5].name "vespa.type.attribute.t2" +rankprofile[1].fef.property[5].value "tensor<float>(x{})" +rankprofile[2].name "test" +rankprofile[2].fef.property[0].name "rankingExpression(my_square@31852fecfab75f29).rankingScript" +rankprofile[2].fef.property[0].value "2 * 2" +rankprofile[2].fef.property[1].name "rankingExpression(my_square@56bfa257b4b447a2).rankingScript" +rankprofile[2].fef.property[1].value "-3.14 * -3.14" +rankprofile[2].fef.property[2].name "rankingExpression(test_constant).rankingScript" +rankprofile[2].fef.property[2].value "rankingExpression(my_square@31852fecfab75f29) + rankingExpression(my_square@56bfa257b4b447a2)" +rankprofile[2].fef.property[3].name "rankingExpression(autogenerated_ranking_feature@c76aeb97d0b6610c).rankingScript" +rankprofile[2].fef.property[3].value "attribute(i1) * 2" +rankprofile[2].fef.property[4].name "rankingExpression(my_square@52d8ef83411bc8d0).rankingScript" +rankprofile[2].fef.property[4].value "rankingExpression(autogenerated_ranking_feature@c76aeb97d0b6610c) * rankingExpression(autogenerated_ranking_feature@c76aeb97d0b6610c)" +rankprofile[2].fef.property[5].name "rankingExpression(autogenerated_ranking_feature@828fd0618dc7fc06).rankingScript" +rankprofile[2].fef.property[5].value "attribute(i1) < 1" +rankprofile[2].fef.property[6].name "rankingExpression(my_square@6b77cba8e7358f11).rankingScript" +rankprofile[2].fef.property[6].value "rankingExpression(autogenerated_ranking_feature@828fd0618dc7fc06) * rankingExpression(autogenerated_ranking_feature@828fd0618dc7fc06)" +rankprofile[2].fef.property[7].name "rankingExpression(test_arithmetic).rankingScript" +rankprofile[2].fef.property[7].value "rankingExpression(my_square@52d8ef83411bc8d0) + rankingExpression(my_square@6b77cba8e7358f11)" +rankprofile[2].fef.property[8].name "rankingExpression(autogenerated_ranking_feature@675b0f8c6790c8bb).rankingScript" +rankprofile[2].fef.property[8].value "!attribute(i1)" +rankprofile[2].fef.property[9].name "rankingExpression(my_square@35879139f3786e2f).rankingScript" +rankprofile[2].fef.property[9].value "rankingExpression(autogenerated_ranking_feature@675b0f8c6790c8bb) * rankingExpression(autogenerated_ranking_feature@675b0f8c6790c8bb)" +rankprofile[2].fef.property[10].name "rankingExpression(autogenerated_ranking_feature@6084beaceb676bf2).rankingScript" +rankprofile[2].fef.property[10].value "-attribute(i1)" +rankprofile[2].fef.property[11].name "rankingExpression(my_square@819381f707f3ee78).rankingScript" +rankprofile[2].fef.property[11].value "rankingExpression(autogenerated_ranking_feature@6084beaceb676bf2) * rankingExpression(autogenerated_ranking_feature@6084beaceb676bf2)" +rankprofile[2].fef.property[12].name "rankingExpression(test_not_neg).rankingScript" +rankprofile[2].fef.property[12].value "rankingExpression(my_square@35879139f3786e2f) + rankingExpression(my_square@819381f707f3ee78)" +rankprofile[2].fef.property[13].name "rankingExpression(autogenerated_ranking_feature@b41d2fa3c2ee40a3).rankingScript" +rankprofile[2].fef.property[13].value "if (attribute(i1) in [0, 1, 2], 0, 1)" +rankprofile[2].fef.property[14].name "rankingExpression(my_square@643af1a7339a8b1b).rankingScript" +rankprofile[2].fef.property[14].value "rankingExpression(autogenerated_ranking_feature@b41d2fa3c2ee40a3) * rankingExpression(autogenerated_ranking_feature@b41d2fa3c2ee40a3)" +rankprofile[2].fef.property[15].name "rankingExpression(autogenerated_ranking_feature@335f7caa94692e97).rankingScript" +rankprofile[2].fef.property[15].value "attribute(i1) in [0, 1, 2]" +rankprofile[2].fef.property[16].name "rankingExpression(my_square@d48185ac029647a5).rankingScript" +rankprofile[2].fef.property[16].value "rankingExpression(autogenerated_ranking_feature@335f7caa94692e97) * rankingExpression(autogenerated_ranking_feature@335f7caa94692e97)" +rankprofile[2].fef.property[17].name "rankingExpression(test_if_in).rankingScript" +rankprofile[2].fef.property[17].value "rankingExpression(my_square@643af1a7339a8b1b) + rankingExpression(my_square@d48185ac029647a5)" +rankprofile[2].fef.property[18].name "rankingExpression(my_square@9a5117ae5a6d491b).rankingScript" +rankprofile[2].fef.property[18].value "cos(attribute(i1)) * cos(attribute(i1))" +rankprofile[2].fef.property[19].name "rankingExpression(test_function).rankingScript" +rankprofile[2].fef.property[19].value "rankingExpression(my_square@9a5117ae5a6d491b)" +rankprofile[2].fef.property[20].name "rankingExpression(autogenerated_ranking_feature@6c8232a0cd94322d).rankingScript" +rankprofile[2].fef.property[20].value "(attribute(i1) * 2)" +rankprofile[2].fef.property[21].name "rankingExpression(my_square@181aa0cc505c1788).rankingScript" +rankprofile[2].fef.property[21].value "rankingExpression(autogenerated_ranking_feature@6c8232a0cd94322d) * rankingExpression(autogenerated_ranking_feature@6c8232a0cd94322d)" +rankprofile[2].fef.property[22].name "rankingExpression(test_embraced).rankingScript" +rankprofile[2].fef.property[22].value "rankingExpression(my_square@181aa0cc505c1788)" +rankprofile[2].fef.property[23].name "rankingExpression(my_func@9bbaee2bad5a2fc0).rankingScript" +rankprofile[2].fef.property[23].value "reduce(attribute(t1), sum, x) + 1" +rankprofile[2].fef.property[24].name "rankingExpression(test_func).rankingScript" +rankprofile[2].fef.property[24].value "rankingExpression(my_func@9bbaee2bad5a2fc0)" +rankprofile[2].fef.property[25].name "rankingExpression(autogenerated_ranking_feature@43bc412603c00a4a).rankingScript" +rankprofile[2].fef.property[25].value "attribute(t1) * attribute(t2)" +rankprofile[2].fef.property[26].name "rankingExpression(my_func@7f288a910482845a).rankingScript" +rankprofile[2].fef.property[26].value "reduce(rankingExpression(autogenerated_ranking_feature@43bc412603c00a4a), sum, x) + 1" +rankprofile[2].fef.property[27].name "rankingExpression(test_tensor_func_with_expr).rankingScript" +rankprofile[2].fef.property[27].value "rankingExpression(my_func@7f288a910482845a)" +rankprofile[2].fef.property[28].name "rankingExpression(autogenerated_ranking_feature@c1057dea8228da3a).rankingScript" +rankprofile[2].fef.property[28].value "map(attribute(t1), f(x)(x * x))" +rankprofile[2].fef.property[29].name "rankingExpression(my_func@901c2cc6ceb37765).rankingScript" +rankprofile[2].fef.property[29].value "reduce(rankingExpression(autogenerated_ranking_feature@c1057dea8228da3a), sum, x) + 1" +rankprofile[2].fef.property[30].name "rankingExpression(test_func_with_tensor_func).rankingScript" +rankprofile[2].fef.property[30].value "rankingExpression(my_func@901c2cc6ceb37765)" +rankprofile[2].fef.property[31].name "rankingExpression(autogenerated_ranking_feature@fb1a4642f23d9d05).rankingScript" +rankprofile[2].fef.property[31].value "attribute(t1){x:0}" +rankprofile[2].fef.property[32].name "rankingExpression(my_square@1d27f1b495b50910).rankingScript" +rankprofile[2].fef.property[32].value "rankingExpression(autogenerated_ranking_feature@fb1a4642f23d9d05) * rankingExpression(autogenerated_ranking_feature@fb1a4642f23d9d05)" +rankprofile[2].fef.property[33].name "rankingExpression(test_func_with_slice).rankingScript" +rankprofile[2].fef.property[33].value "rankingExpression(my_square@1d27f1b495b50910)" +rankprofile[2].fef.property[34].name "rankingExpression(call_func_with_expr@640470df47a83000.c156faa8f98c0b0c).rankingScript" +rankprofile[2].fef.property[34].value "rankingExpression(my_func@7f288a910482845a)" +rankprofile[2].fef.property[35].name "rankingExpression(test_func_via_func_with_expr).rankingScript" +rankprofile[2].fef.property[35].value "rankingExpression(call_func_with_expr@640470df47a83000.c156faa8f98c0b0c)" +rankprofile[2].fef.property[36].name "rankingExpression(my_square).rankingScript" +rankprofile[2].fef.property[36].value "x * x" +rankprofile[2].fef.property[37].name "rankingExpression(my_func).rankingScript" +rankprofile[2].fef.property[37].value "reduce(t, sum, x) + 1" +rankprofile[2].fef.property[38].name "rankingExpression(autogenerated_ranking_feature@1044065d971a7507).rankingScript" +rankprofile[2].fef.property[38].value "a * b" +rankprofile[2].fef.property[39].name "rankingExpression(my_func@93366be10bade547).rankingScript" +rankprofile[2].fef.property[39].value "reduce(rankingExpression(autogenerated_ranking_feature@1044065d971a7507), sum, x) + 1" +rankprofile[2].fef.property[40].name "rankingExpression(call_func_with_expr).rankingScript" +rankprofile[2].fef.property[40].value "rankingExpression(my_func@93366be10bade547)" +rankprofile[2].fef.property[41].name "vespa.rank.firstphase" +rankprofile[2].fef.property[41].value "rankingExpression(firstphase)" +rankprofile[2].fef.property[42].name "rankingExpression(firstphase).rankingScript" +rankprofile[2].fef.property[42].value "42" +rankprofile[2].fef.property[43].name "vespa.summary.feature" +rankprofile[2].fef.property[43].value "rankingExpression(test_constant)" +rankprofile[2].fef.property[44].name "vespa.summary.feature" +rankprofile[2].fef.property[44].value "rankingExpression(test_arithmetic)" +rankprofile[2].fef.property[45].name "vespa.summary.feature" +rankprofile[2].fef.property[45].value "rankingExpression(test_not_neg)" +rankprofile[2].fef.property[46].name "vespa.summary.feature" +rankprofile[2].fef.property[46].value "rankingExpression(test_if_in)" +rankprofile[2].fef.property[47].name "vespa.summary.feature" +rankprofile[2].fef.property[47].value "rankingExpression(test_function)" +rankprofile[2].fef.property[48].name "vespa.summary.feature" +rankprofile[2].fef.property[48].value "rankingExpression(test_embraced)" +rankprofile[2].fef.property[49].name "vespa.summary.feature" +rankprofile[2].fef.property[49].value "rankingExpression(test_func)" +rankprofile[2].fef.property[50].name "vespa.summary.feature" +rankprofile[2].fef.property[50].value "rankingExpression(test_tensor_func_with_expr)" +rankprofile[2].fef.property[51].name "vespa.summary.feature" +rankprofile[2].fef.property[51].value "rankingExpression(test_func_with_tensor_func)" +rankprofile[2].fef.property[52].name "vespa.summary.feature" +rankprofile[2].fef.property[52].value "rankingExpression(test_func_with_slice)" +rankprofile[2].fef.property[53].name "vespa.summary.feature" +rankprofile[2].fef.property[53].value "rankingExpression(test_func_via_func_with_expr)" +rankprofile[2].fef.property[54].name "vespa.feature.rename" +rankprofile[2].fef.property[54].value "rankingExpression(test_constant)" +rankprofile[2].fef.property[55].name "vespa.feature.rename" +rankprofile[2].fef.property[55].value "test_constant" +rankprofile[2].fef.property[56].name "vespa.feature.rename" +rankprofile[2].fef.property[56].value "rankingExpression(test_arithmetic)" +rankprofile[2].fef.property[57].name "vespa.feature.rename" +rankprofile[2].fef.property[57].value "test_arithmetic" +rankprofile[2].fef.property[58].name "vespa.feature.rename" +rankprofile[2].fef.property[58].value "rankingExpression(test_not_neg)" +rankprofile[2].fef.property[59].name "vespa.feature.rename" +rankprofile[2].fef.property[59].value "test_not_neg" +rankprofile[2].fef.property[60].name "vespa.feature.rename" +rankprofile[2].fef.property[60].value "rankingExpression(test_if_in)" +rankprofile[2].fef.property[61].name "vespa.feature.rename" +rankprofile[2].fef.property[61].value "test_if_in" +rankprofile[2].fef.property[62].name "vespa.feature.rename" +rankprofile[2].fef.property[62].value "rankingExpression(test_function)" +rankprofile[2].fef.property[63].name "vespa.feature.rename" +rankprofile[2].fef.property[63].value "test_function" +rankprofile[2].fef.property[64].name "vespa.feature.rename" +rankprofile[2].fef.property[64].value "rankingExpression(test_embraced)" +rankprofile[2].fef.property[65].name "vespa.feature.rename" +rankprofile[2].fef.property[65].value "test_embraced" +rankprofile[2].fef.property[66].name "vespa.feature.rename" +rankprofile[2].fef.property[66].value "rankingExpression(test_func)" +rankprofile[2].fef.property[67].name "vespa.feature.rename" +rankprofile[2].fef.property[67].value "test_func" +rankprofile[2].fef.property[68].name "vespa.feature.rename" +rankprofile[2].fef.property[68].value "rankingExpression(test_tensor_func_with_expr)" +rankprofile[2].fef.property[69].name "vespa.feature.rename" +rankprofile[2].fef.property[69].value "test_tensor_func_with_expr" +rankprofile[2].fef.property[70].name "vespa.feature.rename" +rankprofile[2].fef.property[70].value "rankingExpression(test_func_with_tensor_func)" +rankprofile[2].fef.property[71].name "vespa.feature.rename" +rankprofile[2].fef.property[71].value "test_func_with_tensor_func" +rankprofile[2].fef.property[72].name "vespa.feature.rename" +rankprofile[2].fef.property[72].value "rankingExpression(test_func_with_slice)" +rankprofile[2].fef.property[73].name "vespa.feature.rename" +rankprofile[2].fef.property[73].value "test_func_with_slice" +rankprofile[2].fef.property[74].name "vespa.feature.rename" +rankprofile[2].fef.property[74].value "rankingExpression(test_func_via_func_with_expr)" +rankprofile[2].fef.property[75].name "vespa.feature.rename" +rankprofile[2].fef.property[75].value "test_func_via_func_with_expr" +rankprofile[2].fef.property[76].name "vespa.type.attribute.t1" +rankprofile[2].fef.property[76].value "tensor<float>(x{})" +rankprofile[2].fef.property[77].name "vespa.type.attribute.t2" +rankprofile[2].fef.property[77].value "tensor<float>(x{})" diff --git a/model-evaluation/src/test/resources/config/expressions-as-arguments/ranking-constants.cfg b/model-evaluation/src/test/resources/config/expressions-as-arguments/ranking-constants.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/model-evaluation/src/test/resources/config/expressions-as-arguments/ranking-constants.cfg diff --git a/model-evaluation/src/test/resources/config/expressions-as-arguments/ranking-expressions.cfg b/model-evaluation/src/test/resources/config/expressions-as-arguments/ranking-expressions.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/model-evaluation/src/test/resources/config/expressions-as-arguments/ranking-expressions.cfg diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read.tbf Binary files differindex bac75f7b1e7..bac75f7b1e7 100644 --- a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read.tbf diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read.tbf Binary files differindex bd3f05be826..bd3f05be826 100644 --- a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read.tbf diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read.tbf Binary files differindex fca7c76df3f..fca7c76df3f 100644 --- a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read.tbf diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read.tbf Binary files differindex 396dea8f4bc..396dea8f4bc 100644 --- a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read.tbf diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read.tbf Binary files differindex 42f85478c10..42f85478c10 100644 --- a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read.tbf diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read.tbf Binary files differindex a3cc7d765f6..a3cc7d765f6 100644 --- a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read.tbf diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable.tbf Binary files differindex e768328bff5..e768328bff5 100644 --- a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable.tbf diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1 b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1.tbf Binary files differindex 4fa0eadb0d3..4fa0eadb0d3 100644 --- a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1 +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1.tbf diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read.tbf Binary files differindex 4fa0eadb0d3..4fa0eadb0d3 100644 --- a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read.tbf diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read.tbf Binary files differindex e768328bff5..e768328bff5 100644 --- a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read.tbf diff --git a/model-evaluation/src/test/resources/config/models/ranking-constants.cfg b/model-evaluation/src/test/resources/config/models/ranking-constants.cfg index 2b7495ace5e..335a7ec526b 100644 --- a/model-evaluation/src/test/resources/config/models/ranking-constants.cfg +++ b/model-evaluation/src/test/resources/config/models/ranking-constants.cfg @@ -1,30 +1,30 @@ constant[0].name "mnist_saved_dnn_hidden1_weights_read" -constant[0].fileref "" +constant[0].fileref "constants/mnist_saved_dnn_hidden1_weights_read.tbf" constant[0].type "tensor(d3[300],d4[784])" constant[1].name "mnist_saved_dnn_hidden2_weights_read" -constant[1].fileref "" +constant[1].fileref "constants/mnist_saved_dnn_hidden2_weights_read.tbf" constant[1].type "tensor(d2[100],d3[300])" constant[2].name "mnist_softmax_saved_layer_Variable_1_read" -constant[2].fileref "" +constant[2].fileref "constants/mnist_softmax_saved_layer_Variable_1_read.tbf" constant[2].type "tensor(d1[10])" constant[3].name "mnist_saved_dnn_hidden1_bias_read" -constant[3].fileref "" +constant[3].fileref "constants/mnist_saved_dnn_hidden1_bias_read.tbf" constant[3].type "tensor(d3[300])" constant[4].name "mnist_saved_dnn_hidden2_bias_read" -constant[4].fileref "" +constant[4].fileref "constants/mnist_saved_dnn_hidden2_bias_read.tbf" constant[4].type "tensor(d2[100])" constant[5].name "mnist_softmax_Variable" -constant[5].fileref "" +constant[5].fileref "constants/mnist_softmax_Variable.tbf" constant[5].type "tensor(d1[10],d2[784])" constant[6].name "mnist_saved_dnn_outputs_weights_read" -constant[6].fileref "" +constant[6].fileref "constants/mnist_saved_dnn_outputs_weights_read.tbf" constant[6].type "tensor(d1[10],d2[100])" constant[7].name "mnist_softmax_saved_layer_Variable_read" -constant[7].fileref "" +constant[7].fileref "constants/mnist_softmax_saved_layer_Variable_read.tbf" constant[7].type "tensor(d1[10],d2[784])" constant[8].name "mnist_softmax_Variable_1" -constant[8].fileref "" +constant[8].fileref "constants/mnist_softmax_Variable_1.tbf" constant[8].type "tensor(d1[10])" constant[9].name "mnist_saved_dnn_outputs_bias_read" -constant[9].fileref "" -constant[9].type "tensor(d1[10])"
\ No newline at end of file +constant[9].fileref "constants/mnist_saved_dnn_outputs_bias_read.tbf" +constant[9].type "tensor(d1[10])" diff --git a/model-evaluation/src/test/resources/config/rankexpression/constants/overflow.firstphase.expr b/model-evaluation/src/test/resources/config/rankexpression/expressions/overflow.firstphase.expr index 70a3a2eb6cc..70a3a2eb6cc 100644 --- a/model-evaluation/src/test/resources/config/rankexpression/constants/overflow.firstphase.expr +++ b/model-evaluation/src/test/resources/config/rankexpression/expressions/overflow.firstphase.expr diff --git a/model-evaluation/src/test/resources/config/rankexpression/constants/overflow.firstphase.expr.lz4 b/model-evaluation/src/test/resources/config/rankexpression/expressions/overflow.firstphase.expr.lz4 Binary files differindex 30f23b963db..30f23b963db 100644 --- a/model-evaluation/src/test/resources/config/rankexpression/constants/overflow.firstphase.expr.lz4 +++ b/model-evaluation/src/test/resources/config/rankexpression/expressions/overflow.firstphase.expr.lz4 diff --git a/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg b/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg index 8cb02567538..e8fde6fafb2 100644 --- a/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg +++ b/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg @@ -1,4 +1,4 @@ expression[0].name "overflow.firstphase" -expression[0].fileref "overflow.firstphase.expr" +expression[0].fileref "expressions/overflow.firstphase.expr" expression[1].name "overflow.secondphase" -expression[1].fileref "overflow.firstphase.expr.lz4" +expression[1].fileref "expressions/overflow.firstphase.expr.lz4" diff --git a/model-evaluation/src/test/resources/config/ranking-macros/onnx-models.cfg b/model-evaluation/src/test/resources/config/ranking-macros/onnx-models.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/model-evaluation/src/test/resources/config/ranking-macros/onnx-models.cfg diff --git a/model-evaluation/src/test/resources/config/ranking-macros/rank-profiles.cfg b/model-evaluation/src/test/resources/config/ranking-macros/rank-profiles.cfg new file mode 100644 index 00000000000..2f1657aebf3 --- /dev/null +++ b/model-evaluation/src/test/resources/config/ranking-macros/rank-profiles.cfg @@ -0,0 +1,83 @@ +rankprofile[0].name "default" +rankprofile[1].name "unranked" +rankprofile[1].fef.property[0].name "vespa.rank.firstphase" +rankprofile[1].fef.property[0].value "value(0)" +rankprofile[1].fef.property[1].name "vespa.hitcollector.heapsize" +rankprofile[1].fef.property[1].value "0" +rankprofile[1].fef.property[2].name "vespa.hitcollector.arraysize" +rankprofile[1].fef.property[2].value "0" +rankprofile[1].fef.property[3].name "vespa.dump.ignoredefaultfeatures" +rankprofile[1].fef.property[3].value "true" +rankprofile[2].name "standalone" +rankprofile[2].fef.property[0].name "rankingExpression(myfeature).rankingScript" +rankprofile[2].fef.property[0].value "7 * attribute(num)" +rankprofile[2].fef.property[1].name "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86).rankingScript" +rankprofile[2].fef.property[1].value "4 * (match + match)" +rankprofile[2].fef.property[2].name "rankingExpression(macro_with_dollar$).rankingScript" +rankprofile[2].fef.property[2].value "69" +rankprofile[2].fef.property[3].name "rankingExpression(anotherfeature).rankingScript" +rankprofile[2].fef.property[3].value "10 * rankingExpression(myfeature)" +rankprofile[2].fef.property[4].name "rankingExpression(yetanotherfeature).rankingScript" +rankprofile[2].fef.property[4].value "100 * rankingExpression(myfeature)" +rankprofile[2].fef.property[5].name "rankingExpression(fourtimessum).rankingScript" +rankprofile[2].fef.property[5].value "4 * (var1 + var2)" +rankprofile[2].fef.property[6].name "vespa.rank.firstphase" +rankprofile[2].fef.property[6].value "rankingExpression(firstphase)" +rankprofile[2].fef.property[7].name "rankingExpression(firstphase).rankingScript" +rankprofile[2].fef.property[7].value "match + fieldMatch(title) + rankingExpression(myfeature)" +rankprofile[2].fef.property[8].name "vespa.rank.secondphase" +rankprofile[2].fef.property[8].value "rankingExpression(secondphase)" +rankprofile[2].fef.property[9].name "rankingExpression(secondphase).rankingScript" +rankprofile[2].fef.property[9].value "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86) + 0 * rankingExpression(macro_with_dollar$)" +rankprofile[2].fef.property[10].name "vespa.summary.feature" +rankprofile[2].fef.property[10].value "firstPhase" +rankprofile[2].fef.property[11].name "vespa.summary.feature" +rankprofile[2].fef.property[11].value "rankingExpression(myfeature)" +rankprofile[2].fef.property[12].name "vespa.summary.feature" +rankprofile[2].fef.property[12].value "rankingExpression(anotherfeature)" +rankprofile[2].fef.property[13].name "vespa.summary.feature" +rankprofile[2].fef.property[13].value "rankingExpression(yetanotherfeature)" +rankprofile[2].fef.property[14].name "vespa.summary.feature" +rankprofile[2].fef.property[14].value "rankingExpression(macro_with_dollar$)" +rankprofile[2].fef.property[15].name "vespa.feature.rename" +rankprofile[2].fef.property[15].value "rankingExpression(anotherfeature)" +rankprofile[2].fef.property[16].name "vespa.feature.rename" +rankprofile[2].fef.property[16].value "anotherfeature" +rankprofile[2].fef.property[17].name "vespa.feature.rename" +rankprofile[2].fef.property[17].value "rankingExpression(yetanotherfeature)" +rankprofile[2].fef.property[18].name "vespa.feature.rename" +rankprofile[2].fef.property[18].value "yetanotherfeature" +rankprofile[2].fef.property[19].name "vespa.feature.rename" +rankprofile[2].fef.property[19].value "rankingExpression(macro_with_dollar$)" +rankprofile[2].fef.property[20].name "vespa.feature.rename" +rankprofile[2].fef.property[20].value "macro_with_dollar$" +rankprofile[3].name "constantsAndMacro" +rankprofile[3].fef.property[0].name "rankingExpression(c).rankingScript" +rankprofile[3].fef.property[0].value "attribute(num)" +rankprofile[3].fef.property[1].name "vespa.rank.firstphase" +rankprofile[3].fef.property[1].value "rankingExpression(firstphase)" +rankprofile[3].fef.property[2].name "rankingExpression(firstphase).rankingScript" +rankprofile[3].fef.property[2].value "attribute(num) * 2.0 + 3.0" +rankprofile[3].fef.property[3].name "vespa.summary.feature" +rankprofile[3].fef.property[3].value "firstPhase" +rankprofile[4].name "doc" +rankprofile[4].fef.property[0].name "rankingExpression(myfeature).rankingScript" +rankprofile[4].fef.property[0].value "fieldMatch(title) + freshness(timestamp)" +rankprofile[4].fef.property[1].name "rankingExpression(otherfeature@6b0a229a66fcaa04).rankingScript" +rankprofile[4].fef.property[1].value "nativeRank(title,body)" +rankprofile[4].fef.property[2].name "rankingExpression(otherfeature).rankingScript" +rankprofile[4].fef.property[2].value "nativeRank(foo,body)" +rankprofile[4].fef.property[3].name "vespa.rank.firstphase" +rankprofile[4].fef.property[3].value "rankingExpression(firstphase)" +rankprofile[4].fef.property[4].name "rankingExpression(firstphase).rankingScript" +rankprofile[4].fef.property[4].value "rankingExpression(myfeature) * 10" +rankprofile[4].fef.property[5].name "vespa.rank.secondphase" +rankprofile[4].fef.property[5].value "rankingExpression(secondphase)" +rankprofile[4].fef.property[6].name "rankingExpression(secondphase).rankingScript" +rankprofile[4].fef.property[6].value "rankingExpression(otherfeature@6b0a229a66fcaa04) * rankingExpression(myfeature)" +rankprofile[4].fef.property[7].name "vespa.summary.feature" +rankprofile[4].fef.property[7].value "rankingExpression(myfeature)" +rankprofile[4].fef.property[8].name "vespa.feature.rename" +rankprofile[4].fef.property[8].value "rankingExpression(myfeature)" +rankprofile[4].fef.property[9].name "vespa.feature.rename" +rankprofile[4].fef.property[9].value "myfeature"
\ No newline at end of file diff --git a/model-evaluation/src/test/resources/config/ranking-macros/ranking-constants.cfg b/model-evaluation/src/test/resources/config/ranking-macros/ranking-constants.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/model-evaluation/src/test/resources/config/ranking-macros/ranking-constants.cfg diff --git a/model-evaluation/src/test/resources/config/ranking-macros/ranking-expressions.cfg b/model-evaluation/src/test/resources/config/ranking-macros/ranking-expressions.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/model-evaluation/src/test/resources/config/ranking-macros/ranking-expressions.cfg diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/12c4ee4c5547a64e/tinybert_encoder_layer_3_attention_output_LayerNorm_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/12c4ee4c5547a64e/tinybert_encoder_layer_3_attention_output_LayerNorm_bias.tbf Binary files differnew file mode 100644 index 00000000000..de3c8e5bf5d --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/12c4ee4c5547a64e/tinybert_encoder_layer_3_attention_output_LayerNorm_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/12e62273a701da0c/tinybert_encoder_layer_1_attention_output_LayerNorm_weight.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/12e62273a701da0c/tinybert_encoder_layer_1_attention_output_LayerNorm_weight.tbf Binary files differnew file mode 100644 index 00000000000..fbbda1936b9 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/12e62273a701da0c/tinybert_encoder_layer_1_attention_output_LayerNorm_weight.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/13cb87130508f381/tinybert_encoder_layer_3_output_LayerNorm_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/13cb87130508f381/tinybert_encoder_layer_3_output_LayerNorm_bias.tbf Binary files differnew file mode 100644 index 00000000000..19ed89626dc --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/13cb87130508f381/tinybert_encoder_layer_3_output_LayerNorm_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/14b40ffc665653da/tinybert_embeddings_LayerNorm_weight.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/14b40ffc665653da/tinybert_embeddings_LayerNorm_weight.tbf Binary files differnew file mode 100644 index 00000000000..ac77ab88387 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/14b40ffc665653da/tinybert_embeddings_LayerNorm_weight.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/1667a5cf592c365/tinybert_615.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/1667a5cf592c365/tinybert_615.tbf Binary files differnew file mode 100644 index 00000000000..2d5c391f448 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/1667a5cf592c365/tinybert_615.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/16bc5a7698891f15/tinybert_pooler_dense_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/16bc5a7698891f15/tinybert_pooler_dense_bias.tbf Binary files differnew file mode 100644 index 00000000000..548b7688322 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/16bc5a7698891f15/tinybert_pooler_dense_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/1bdfde9445c7cb6d/tinybert_encoder_layer_0_attention_output_LayerNorm_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/1bdfde9445c7cb6d/tinybert_encoder_layer_0_attention_output_LayerNorm_bias.tbf Binary files differnew file mode 100644 index 00000000000..9f1103c1ab8 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/1bdfde9445c7cb6d/tinybert_encoder_layer_0_attention_output_LayerNorm_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/1bfb351718ce25f2/tinybert_Concat_363.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/1bfb351718ce25f2/tinybert_Concat_363.tbf Binary files differnew file mode 100644 index 00000000000..92f5925674a --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/1bfb351718ce25f2/tinybert_Concat_363.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/1e0c4bc8a082d938/tinybert_encoder_layer_1_attention_output_dense_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/1e0c4bc8a082d938/tinybert_encoder_layer_1_attention_output_dense_bias.tbf Binary files differnew file mode 100644 index 00000000000..581f0261c86 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/1e0c4bc8a082d938/tinybert_encoder_layer_1_attention_output_dense_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/24d385c5920aaaec/tinybert_encoder_layer_0_output_dense_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/24d385c5920aaaec/tinybert_encoder_layer_0_output_dense_bias.tbf Binary files differnew file mode 100644 index 00000000000..7da63fd9275 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/24d385c5920aaaec/tinybert_encoder_layer_0_output_dense_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/261b74d9b1faa77d/tinybert_encoder_layer_3_attention_self_query_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/261b74d9b1faa77d/tinybert_encoder_layer_3_attention_self_query_bias.tbf Binary files differnew file mode 100644 index 00000000000..c5f7ea3dc10 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/261b74d9b1faa77d/tinybert_encoder_layer_3_attention_self_query_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/27e2afaa08b38e1/tinybert_encoder_layer_1_output_LayerNorm_weight.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/27e2afaa08b38e1/tinybert_encoder_layer_1_output_LayerNorm_weight.tbf Binary files differnew file mode 100644 index 00000000000..bd5526d35ac --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/27e2afaa08b38e1/tinybert_encoder_layer_1_output_LayerNorm_weight.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/2901f3bcfdb170a4/tinybert_611.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/2901f3bcfdb170a4/tinybert_611.tbf Binary files differnew file mode 100644 index 00000000000..9e03951e83d --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/2901f3bcfdb170a4/tinybert_611.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/31795fa4ab9cdcee/tinybert_encoder_layer_0_output_LayerNorm_weight.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/31795fa4ab9cdcee/tinybert_encoder_layer_0_output_LayerNorm_weight.tbf Binary files differnew file mode 100644 index 00000000000..644535defda --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/31795fa4ab9cdcee/tinybert_encoder_layer_0_output_LayerNorm_weight.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/36b4a3d779ef7201/tinybert_600.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/36b4a3d779ef7201/tinybert_600.tbf Binary files differnew file mode 100644 index 00000000000..5f225f99f18 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/36b4a3d779ef7201/tinybert_600.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/3a3d747f48b9d8c7/tinybert_encoder_layer_0_output_LayerNorm_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/3a3d747f48b9d8c7/tinybert_encoder_layer_0_output_LayerNorm_bias.tbf Binary files differnew file mode 100644 index 00000000000..8e46c9c29e3 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/3a3d747f48b9d8c7/tinybert_encoder_layer_0_output_LayerNorm_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/4f7d38d29831c94c/tinybert_Concat_269.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/4f7d38d29831c94c/tinybert_Concat_269.tbf Binary files differnew file mode 100644 index 00000000000..92f5925674a --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/4f7d38d29831c94c/tinybert_Concat_269.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/52e4f13729329216/tinybert_624.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/52e4f13729329216/tinybert_624.tbf Binary files differnew file mode 100644 index 00000000000..992f315f25f --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/52e4f13729329216/tinybert_624.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/5326aaa30d3cd2fd/tinybert_encoder_layer_2_attention_output_LayerNorm_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/5326aaa30d3cd2fd/tinybert_encoder_layer_2_attention_output_LayerNorm_bias.tbf Binary files differnew file mode 100644 index 00000000000..7811770c682 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/5326aaa30d3cd2fd/tinybert_encoder_layer_2_attention_output_LayerNorm_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/5571e3625094b475/tinybert_629.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/5571e3625094b475/tinybert_629.tbf Binary files differnew file mode 100644 index 00000000000..3ca924832e2 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/5571e3625094b475/tinybert_629.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/590fe4d33400f007/tinybert_embeddings_LayerNorm_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/590fe4d33400f007/tinybert_embeddings_LayerNorm_bias.tbf Binary files differnew file mode 100644 index 00000000000..ca7f0a68c87 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/590fe4d33400f007/tinybert_embeddings_LayerNorm_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/5bdfe53fc08f095/tinybert_encoder_layer_0_attention_self_query_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/5bdfe53fc08f095/tinybert_encoder_layer_0_attention_self_query_bias.tbf Binary files differnew file mode 100644 index 00000000000..b4a7ae93485 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/5bdfe53fc08f095/tinybert_encoder_layer_0_attention_self_query_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/5d4350323a071e9f/tinybert_616.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/5d4350323a071e9f/tinybert_616.tbf Binary files differnew file mode 100644 index 00000000000..227eddbb104 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/5d4350323a071e9f/tinybert_616.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/5f4e10876d3b273b/tinybert_630.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/5f4e10876d3b273b/tinybert_630.tbf Binary files differnew file mode 100644 index 00000000000..c807c7f8afb --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/5f4e10876d3b273b/tinybert_630.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/5fa74195044c3fcd/tinybert_Concat_154.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/5fa74195044c3fcd/tinybert_Concat_154.tbf Binary files differnew file mode 100644 index 00000000000..92f5925674a --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/5fa74195044c3fcd/tinybert_Concat_154.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/60bab4fc23fe183e/tinybert_Concat_248.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/60bab4fc23fe183e/tinybert_Concat_248.tbf Binary files differnew file mode 100644 index 00000000000..92f5925674a --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/60bab4fc23fe183e/tinybert_Concat_248.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/61115ebdbd912cc1/tinybert_encoder_layer_0_attention_self_key_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/61115ebdbd912cc1/tinybert_encoder_layer_0_attention_self_key_bias.tbf Binary files differnew file mode 100644 index 00000000000..6c2ce73850d --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/61115ebdbd912cc1/tinybert_encoder_layer_0_attention_self_key_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/62013069bfe7e037/tinybert_631.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/62013069bfe7e037/tinybert_631.tbf Binary files differnew file mode 100644 index 00000000000..6a1664a8aa5 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/62013069bfe7e037/tinybert_631.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/64ea58b13d6a706c/tinybert_Concat_100.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/64ea58b13d6a706c/tinybert_Concat_100.tbf Binary files differnew file mode 100644 index 00000000000..087437be568 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/64ea58b13d6a706c/tinybert_Concat_100.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/65461c703dbdae0c/tinybert_Concat_165.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/65461c703dbdae0c/tinybert_Concat_165.tbf Binary files differnew file mode 100644 index 00000000000..92f5925674a --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/65461c703dbdae0c/tinybert_Concat_165.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/669c970d89cc2ba7/tinybert_Concat_259.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/669c970d89cc2ba7/tinybert_Concat_259.tbf Binary files differnew file mode 100644 index 00000000000..92f5925674a --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/669c970d89cc2ba7/tinybert_Concat_259.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/689165a57d7656c9/tinybert_614.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/689165a57d7656c9/tinybert_614.tbf Binary files differnew file mode 100644 index 00000000000..8eef245c63a --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/689165a57d7656c9/tinybert_614.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/69bd5a278268d6a8/tinybert_encoder_layer_2_attention_self_key_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/69bd5a278268d6a8/tinybert_encoder_layer_2_attention_self_key_bias.tbf Binary files differnew file mode 100644 index 00000000000..aef2fbcf48f --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/69bd5a278268d6a8/tinybert_encoder_layer_2_attention_self_key_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/6b9c014f349b42b3/tinybert_Concat_175.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/6b9c014f349b42b3/tinybert_Concat_175.tbf Binary files differnew file mode 100644 index 00000000000..92f5925674a --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/6b9c014f349b42b3/tinybert_Concat_175.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/704d97f73a6ea9fb/tinybert_encoder_layer_1_attention_self_query_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/704d97f73a6ea9fb/tinybert_encoder_layer_1_attention_self_query_bias.tbf Binary files differnew file mode 100644 index 00000000000..0a29fd69941 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/704d97f73a6ea9fb/tinybert_encoder_layer_1_attention_self_query_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/71379a41f448366f/tinybert_encoder_layer_2_output_LayerNorm_weight.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/71379a41f448366f/tinybert_encoder_layer_2_output_LayerNorm_weight.tbf Binary files differnew file mode 100644 index 00000000000..2cf47343708 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/71379a41f448366f/tinybert_encoder_layer_2_output_LayerNorm_weight.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/767334a7ff3e73e7/tinybert_encoder_layer_1_attention_self_key_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/767334a7ff3e73e7/tinybert_encoder_layer_1_attention_self_key_bias.tbf Binary files differnew file mode 100644 index 00000000000..20fd777fd4d --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/767334a7ff3e73e7/tinybert_encoder_layer_1_attention_self_key_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/7c076b59f45223ef/tinybert_Concat_382.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/7c076b59f45223ef/tinybert_Concat_382.tbf Binary files differnew file mode 100644 index 00000000000..087437be568 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/7c076b59f45223ef/tinybert_Concat_382.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/7d01968ada002ff8/tinybert_embeddings_token_type_embeddings_weight.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/7d01968ada002ff8/tinybert_embeddings_token_type_embeddings_weight.tbf Binary files differnew file mode 100644 index 00000000000..4836206f87b --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/7d01968ada002ff8/tinybert_embeddings_token_type_embeddings_weight.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/7ef6de3dd8902534/tinybert_645.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/7ef6de3dd8902534/tinybert_645.tbf Binary files differnew file mode 100644 index 00000000000..1ca37d183d4 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/7ef6de3dd8902534/tinybert_645.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/8216d52cc5aa31bd/tinybert_Concat_288.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/8216d52cc5aa31bd/tinybert_Concat_288.tbf Binary files differnew file mode 100644 index 00000000000..087437be568 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/8216d52cc5aa31bd/tinybert_Concat_288.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/82f078dd5493eab/tinybert_644.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/82f078dd5493eab/tinybert_644.tbf Binary files differnew file mode 100644 index 00000000000..1b756e25a3c --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/82f078dd5493eab/tinybert_644.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/86bda18083e89158/tinybert_encoder_layer_3_attention_self_key_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/86bda18083e89158/tinybert_encoder_layer_3_attention_self_key_bias.tbf Binary files differnew file mode 100644 index 00000000000..8edf36a983b --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/86bda18083e89158/tinybert_encoder_layer_3_attention_self_key_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/880243d76119d1be/tinybert_Concat_60.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/880243d76119d1be/tinybert_Concat_60.tbf Binary files differnew file mode 100644 index 00000000000..92f5925674a --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/880243d76119d1be/tinybert_Concat_60.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/8ee0f683d1a850c5/tinybert_encoder_layer_3_output_LayerNorm_weight.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/8ee0f683d1a850c5/tinybert_encoder_layer_3_output_LayerNorm_weight.tbf Binary files differnew file mode 100644 index 00000000000..038ae981313 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/8ee0f683d1a850c5/tinybert_encoder_layer_3_output_LayerNorm_weight.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/8fef598bac28930b/tinybert_encoder_layer_2_intermediate_dense_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/8fef598bac28930b/tinybert_encoder_layer_2_intermediate_dense_bias.tbf Binary files differnew file mode 100644 index 00000000000..cc6b08d778f --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/8fef598bac28930b/tinybert_encoder_layer_2_intermediate_dense_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/923e51a03f5d6c88/tinybert_646.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/923e51a03f5d6c88/tinybert_646.tbf Binary files differnew file mode 100644 index 00000000000..29ead9a3353 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/923e51a03f5d6c88/tinybert_646.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/93736ee6488d12f9/tinybert_encoder_layer_2_attention_self_value_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/93736ee6488d12f9/tinybert_encoder_layer_2_attention_self_value_bias.tbf Binary files differnew file mode 100644 index 00000000000..e7012d8bc68 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/93736ee6488d12f9/tinybert_encoder_layer_2_attention_self_value_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/9496f2740a93d981/tinybert_encoder_layer_0_attention_self_value_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/9496f2740a93d981/tinybert_encoder_layer_0_attention_self_value_bias.tbf Binary files differnew file mode 100644 index 00000000000..85f6cccd5d0 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/9496f2740a93d981/tinybert_encoder_layer_0_attention_self_value_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/99f6d1ba780ffe21/tinybert_encoder_layer_2_attention_output_dense_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/99f6d1ba780ffe21/tinybert_encoder_layer_2_attention_output_dense_bias.tbf Binary files differnew file mode 100644 index 00000000000..3c48f3730fe --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/99f6d1ba780ffe21/tinybert_encoder_layer_2_attention_output_dense_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/9a8e585e32950ecf/tinybert_encoder_layer_3_attention_output_dense_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/9a8e585e32950ecf/tinybert_encoder_layer_3_attention_output_dense_bias.tbf Binary files differnew file mode 100644 index 00000000000..7403e39f075 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/9a8e585e32950ecf/tinybert_encoder_layer_3_attention_output_dense_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/9ad03013d8ea199b/tinybert_encoder_layer_0_intermediate_dense_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/9ad03013d8ea199b/tinybert_encoder_layer_0_intermediate_dense_bias.tbf Binary files differnew file mode 100644 index 00000000000..80e48b2ead1 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/9ad03013d8ea199b/tinybert_encoder_layer_0_intermediate_dense_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/9d8e9504a5b2d19b/tinybert_627.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/9d8e9504a5b2d19b/tinybert_627.tbf Binary files differnew file mode 100644 index 00000000000..b03f3622129 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/9d8e9504a5b2d19b/tinybert_627.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/9ee8715bbc085dfb/tinybert_657.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/9ee8715bbc085dfb/tinybert_657.tbf Binary files differnew file mode 100644 index 00000000000..c04c5315e1f --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/9ee8715bbc085dfb/tinybert_657.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/a05168f0aef055b4/tinybert_pooler_dense_weight.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/a05168f0aef055b4/tinybert_pooler_dense_weight.tbf Binary files differnew file mode 100644 index 00000000000..3dd4292318d --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/a05168f0aef055b4/tinybert_pooler_dense_weight.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/a10cda99920a91b7/tinybert_encoder_layer_3_attention_self_value_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/a10cda99920a91b7/tinybert_encoder_layer_3_attention_self_value_bias.tbf Binary files differnew file mode 100644 index 00000000000..a870847c54c --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/a10cda99920a91b7/tinybert_encoder_layer_3_attention_self_value_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/a25a5243e3c26732/tinybert_Gather_31.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/a25a5243e3c26732/tinybert_Gather_31.tbf Binary files differnew file mode 100644 index 00000000000..2bd280993e3 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/a25a5243e3c26732/tinybert_Gather_31.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/a642396691f08a5f/tinybert_encoder_layer_2_attention_self_query_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/a642396691f08a5f/tinybert_encoder_layer_2_attention_self_query_bias.tbf Binary files differnew file mode 100644 index 00000000000..5dedb2c94e4 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/a642396691f08a5f/tinybert_encoder_layer_2_attention_self_query_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/a7d13f6c41a99226/tinybert_641.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/a7d13f6c41a99226/tinybert_641.tbf Binary files differnew file mode 100644 index 00000000000..d62a8785c0e --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/a7d13f6c41a99226/tinybert_641.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/a893d4ac541d42eb/tinybert_encoder_layer_3_intermediate_dense_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/a893d4ac541d42eb/tinybert_encoder_layer_3_intermediate_dense_bias.tbf Binary files differnew file mode 100644 index 00000000000..5eb6b7bc8f8 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/a893d4ac541d42eb/tinybert_encoder_layer_3_intermediate_dense_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/ac91e41746178779/tinybert_encoder_layer_0_attention_output_dense_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/ac91e41746178779/tinybert_encoder_layer_0_attention_output_dense_bias.tbf Binary files differnew file mode 100644 index 00000000000..0cbc07ade5b --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/ac91e41746178779/tinybert_encoder_layer_0_attention_output_dense_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/bac9a3d7f3f7e173/tinybert_encoder_layer_1_output_dense_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/bac9a3d7f3f7e173/tinybert_encoder_layer_1_output_dense_bias.tbf Binary files differnew file mode 100644 index 00000000000..7e6a18a281b --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/bac9a3d7f3f7e173/tinybert_encoder_layer_1_output_dense_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/bb406a628e9df4d3/tinybert_Concat_194.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/bb406a628e9df4d3/tinybert_Concat_194.tbf Binary files differnew file mode 100644 index 00000000000..087437be568 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/bb406a628e9df4d3/tinybert_Concat_194.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/c003ee0fce20531/tinybert_601.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/c003ee0fce20531/tinybert_601.tbf Binary files differnew file mode 100644 index 00000000000..166960d4da2 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/c003ee0fce20531/tinybert_601.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/c0374bdd83f6f831/tinybert_encoder_layer_3_output_dense_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/c0374bdd83f6f831/tinybert_encoder_layer_3_output_dense_bias.tbf Binary files differnew file mode 100644 index 00000000000..8c4c18bce17 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/c0374bdd83f6f831/tinybert_encoder_layer_3_output_dense_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/c165d1ea133a28b6/tinybert_626.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/c165d1ea133a28b6/tinybert_626.tbf Binary files differnew file mode 100644 index 00000000000..9366ca8a7ca --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/c165d1ea133a28b6/tinybert_626.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/c2195a93d2c7aa5c/tinybert_656.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/c2195a93d2c7aa5c/tinybert_656.tbf Binary files differnew file mode 100644 index 00000000000..7ab5931f944 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/c2195a93d2c7aa5c/tinybert_656.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/c5e300a8e998ce7f/tinybert_encoder_layer_1_output_LayerNorm_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/c5e300a8e998ce7f/tinybert_encoder_layer_1_output_LayerNorm_bias.tbf Binary files differnew file mode 100644 index 00000000000..ce70747ae6b --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/c5e300a8e998ce7f/tinybert_encoder_layer_1_output_LayerNorm_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/c6c6f63e836f2582/tinybert_encoder_layer_1_attention_output_LayerNorm_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/c6c6f63e836f2582/tinybert_encoder_layer_1_attention_output_LayerNorm_bias.tbf Binary files differnew file mode 100644 index 00000000000..2c7ee8e296c --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/c6c6f63e836f2582/tinybert_encoder_layer_1_attention_output_LayerNorm_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/ccd015582d66388e/tinybert_encoder_layer_2_output_LayerNorm_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/ccd015582d66388e/tinybert_encoder_layer_2_output_LayerNorm_bias.tbf Binary files differnew file mode 100644 index 00000000000..bffe5b6412e --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/ccd015582d66388e/tinybert_encoder_layer_2_output_LayerNorm_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/cd0d517153ca69b4/tinybert_Concat_81.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/cd0d517153ca69b4/tinybert_Concat_81.tbf Binary files differnew file mode 100644 index 00000000000..92f5925674a --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/cd0d517153ca69b4/tinybert_Concat_81.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/d0392f4c85a475f4/tinybert_encoder_layer_3_attention_output_LayerNorm_weight.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/d0392f4c85a475f4/tinybert_encoder_layer_3_attention_output_LayerNorm_weight.tbf Binary files differnew file mode 100644 index 00000000000..461780b1e6d --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/d0392f4c85a475f4/tinybert_encoder_layer_3_attention_output_LayerNorm_weight.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/d25674a64681be9f/tinybert_encoder_layer_2_output_dense_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/d25674a64681be9f/tinybert_encoder_layer_2_output_dense_bias.tbf Binary files differnew file mode 100644 index 00000000000..093f2aa7b97 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/d25674a64681be9f/tinybert_encoder_layer_2_output_dense_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/db6dae3a39fcf192/tinybert_Concat_342.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/db6dae3a39fcf192/tinybert_Concat_342.tbf Binary files differnew file mode 100644 index 00000000000..92f5925674a --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/db6dae3a39fcf192/tinybert_Concat_342.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/dc72a0e790a30020/tinybert_Concat_353.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/dc72a0e790a30020/tinybert_Concat_353.tbf Binary files differnew file mode 100644 index 00000000000..92f5925674a --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/dc72a0e790a30020/tinybert_Concat_353.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/dfaa3c067cc9a6ff/tinybert_Concat_71.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/dfaa3c067cc9a6ff/tinybert_Concat_71.tbf Binary files differnew file mode 100644 index 00000000000..92f5925674a --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/dfaa3c067cc9a6ff/tinybert_Concat_71.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/e6bf3f48a1e9269e/tinybert_encoder_layer_0_attention_output_LayerNorm_weight.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/e6bf3f48a1e9269e/tinybert_encoder_layer_0_attention_output_LayerNorm_weight.tbf Binary files differnew file mode 100644 index 00000000000..7de9a1d7389 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/e6bf3f48a1e9269e/tinybert_encoder_layer_0_attention_output_LayerNorm_weight.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/e86d7b0aeaea757c/tinybert_609.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/e86d7b0aeaea757c/tinybert_609.tbf Binary files differnew file mode 100644 index 00000000000..b0ed7d20e55 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/e86d7b0aeaea757c/tinybert_609.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/ea71f78349b945f3/tinybert_encoder_layer_1_intermediate_dense_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/ea71f78349b945f3/tinybert_encoder_layer_1_intermediate_dense_bias.tbf Binary files differnew file mode 100644 index 00000000000..9c7a59d0a71 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/ea71f78349b945f3/tinybert_encoder_layer_1_intermediate_dense_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/f063f0b40260827d/tinybert_639.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/f063f0b40260827d/tinybert_639.tbf Binary files differnew file mode 100644 index 00000000000..a135eb0ce4b --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/f063f0b40260827d/tinybert_639.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/f0a05a2cea97e1f9/tinybert_612.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/f0a05a2cea97e1f9/tinybert_612.tbf Binary files differnew file mode 100644 index 00000000000..fbfd8e8b886 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/f0a05a2cea97e1f9/tinybert_612.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/f427490c7481e911/tinybert_encoder_layer_2_attention_output_LayerNorm_weight.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/f427490c7481e911/tinybert_encoder_layer_2_attention_output_LayerNorm_weight.tbf Binary files differnew file mode 100644 index 00000000000..50788184084 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/f427490c7481e911/tinybert_encoder_layer_2_attention_output_LayerNorm_weight.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/f4fd0f62da4d01e4/tinybert_654.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/f4fd0f62da4d01e4/tinybert_654.tbf Binary files differnew file mode 100644 index 00000000000..cee531786fd --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/f4fd0f62da4d01e4/tinybert_654.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/f65178b864c98e80/tinybert_embeddings_word_embeddings_weight.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/f65178b864c98e80/tinybert_embeddings_word_embeddings_weight.tbf Binary files differnew file mode 100644 index 00000000000..ff8192661d4 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/f65178b864c98e80/tinybert_embeddings_word_embeddings_weight.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/f69aa2c5d6e9928d/tinybert_encoder_layer_1_attention_self_value_bias.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/f69aa2c5d6e9928d/tinybert_encoder_layer_1_attention_self_value_bias.tbf Binary files differnew file mode 100644 index 00000000000..d61e4b66e9e --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/f69aa2c5d6e9928d/tinybert_encoder_layer_1_attention_self_value_bias.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/fc60417d2ee421df/tinybert_642.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/fc60417d2ee421df/tinybert_642.tbf Binary files differnew file mode 100644 index 00000000000..07c9d3e82b6 --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/fc60417d2ee421df/tinybert_642.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/filedistribution/ff781c1ad6f97395/tinybert_599.tbf b/model-evaluation/src/test/resources/config/tinybert/filedistribution/ff781c1ad6f97395/tinybert_599.tbf Binary files differnew file mode 100644 index 00000000000..35d73cfb0ae --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/filedistribution/ff781c1ad6f97395/tinybert_599.tbf diff --git a/model-evaluation/src/test/resources/config/tinybert/onnx-models.cfg b/model-evaluation/src/test/resources/config/tinybert/onnx-models.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/onnx-models.cfg diff --git a/model-evaluation/src/test/resources/config/tinybert/rank-profiles.cfg b/model-evaluation/src/test/resources/config/tinybert/rank-profiles.cfg new file mode 100644 index 00000000000..3e422a00c7a --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/rank-profiles.cfg @@ -0,0 +1,161 @@ +rankprofile[0].name "default" +rankprofile[0].fef.property[0].name "vespa.type.attribute.field_token_type_ids" +rankprofile[0].fef.property[0].value "tensor<float>(d0[1],d1[128])" +rankprofile[0].fef.property[1].name "vespa.type.attribute.field_attention_mask" +rankprofile[0].fef.property[1].value "tensor<float>(d0[1],d1[128])" +rankprofile[0].fef.property[2].name "vespa.type.attribute.field_input_ids" +rankprofile[0].fef.property[2].value "tensor<float>(d0[1],d1[128])" +rankprofile[1].name "unranked" +rankprofile[1].fef.property[0].name "vespa.rank.firstphase" +rankprofile[1].fef.property[0].value "value(0)" +rankprofile[1].fef.property[1].name "vespa.hitcollector.heapsize" +rankprofile[1].fef.property[1].value "0" +rankprofile[1].fef.property[2].name "vespa.hitcollector.arraysize" +rankprofile[1].fef.property[2].value "0" +rankprofile[1].fef.property[3].name "vespa.dump.ignoredefaultfeatures" +rankprofile[1].fef.property[3].value "true" +rankprofile[1].fef.property[4].name "vespa.type.attribute.field_token_type_ids" +rankprofile[1].fef.property[4].value "tensor<float>(d0[1],d1[128])" +rankprofile[1].fef.property[5].name "vespa.type.attribute.field_attention_mask" +rankprofile[1].fef.property[5].value "tensor<float>(d0[1],d1[128])" +rankprofile[1].fef.property[6].name "vespa.type.attribute.field_input_ids" +rankprofile[1].fef.property[6].value "tensor<float>(d0[1],d1[128])" +rankprofile[2].name "vespabert" +rankprofile[2].fef.property[0].name "rankingExpression(input_ids).rankingScript" +rankprofile[2].fef.property[0].value "attribute(field_input_ids)" +rankprofile[2].fef.property[1].name "rankingExpression(input_ids).type" +rankprofile[2].fef.property[1].value "tensor<float>(d0[1],d1[128])" +rankprofile[2].fef.property[2].name "rankingExpression(imported_ml_function_tinybert_input_ids).rankingScript" +rankprofile[2].fef.property[2].value "rankingExpression(input_ids)" +rankprofile[2].fef.property[3].name "rankingExpression(token_type_ids).rankingScript" +rankprofile[2].fef.property[3].value "attribute(field_token_type_ids)" +rankprofile[2].fef.property[4].name "rankingExpression(token_type_ids).type" +rankprofile[2].fef.property[4].value "tensor<float>(d0[1],d1[128])" +rankprofile[2].fef.property[5].name "rankingExpression(imported_ml_function_tinybert_token_type_ids).rankingScript" +rankprofile[2].fef.property[5].value "rankingExpression(token_type_ids)" +rankprofile[2].fef.property[6].name "rankingExpression(imported_ml_function_tinybert_Add_34).rankingScript" +rankprofile[2].fef.property[6].value "join(join(tensor<float>(d0[1],d1[128],d3[312])((constant(tinybert_embeddings_word_embeddings_weight){d0:((rankingExpression(imported_ml_function_tinybert_input_ids){d0:(d0), d1:(d1)} + 30522.0) % 30522.0), d3:(d3)})), constant(tinybert_Gather_31), f(a,b)(a + b)), tensor<float>(d0[1],d1[128],d3[312])((constant(tinybert_embeddings_token_type_embeddings_weight){d0:((rankingExpression(imported_ml_function_tinybert_token_type_ids){d0:(d0), d1:(d1)} + 2.0) % 2.0), d3:(d3)})), f(a,b)(a + b))" +rankprofile[2].fef.property[7].name "rankingExpression(imported_ml_function_tinybert_Sub_36).rankingScript" +rankprofile[2].fef.property[7].value "join(rankingExpression(imported_ml_function_tinybert_Add_34), reduce(join(reduce(rankingExpression(imported_ml_function_tinybert_Add_34), avg, d3), tensor<float>(d3[1])(1.0), f(a,b)(a * b)), sum, d3), f(a,b)(a - b))" +rankprofile[2].fef.property[8].name "rankingExpression(imported_ml_function_tinybert_Add_45).rankingScript" +rankprofile[2].fef.property[8].value "join(join(join(rankingExpression(imported_ml_function_tinybert_Sub_36), reduce(map(join(join(reduce(join(rankingExpression(imported_ml_function_tinybert_Sub_36), 2.0, f(a,b)(pow(a,b))), avg, d3), tensor<float>(d3[1])(1.0), f(a,b)(a * b)), 9.999999960041972E-13, f(a,b)(a + b)), f(a)(sqrt(a))), sum, d3), f(a,b)(a / b)), constant(tinybert_embeddings_LayerNorm_weight), f(a,b)(a * b)), constant(tinybert_embeddings_LayerNorm_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[9].name "rankingExpression(imported_ml_function_tinybert_Add_47).rankingScript" +rankprofile[2].fef.property[9].value "join(reduce(join(rankingExpression(imported_ml_function_tinybert_Add_45), constant(tinybert_599), f(a,b)(a * b)), sum, d3), constant(tinybert_encoder_layer_0_attention_self_query_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[10].name "rankingExpression(imported_ml_function_tinybert_Add_49).rankingScript" +rankprofile[2].fef.property[10].value "join(reduce(join(rankingExpression(imported_ml_function_tinybert_Add_45), constant(tinybert_600), f(a,b)(a * b)), sum, d3), constant(tinybert_encoder_layer_0_attention_self_key_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[11].name "rankingExpression(attention_mask).rankingScript" +rankprofile[2].fef.property[11].value "attribute(field_attention_mask)" +rankprofile[2].fef.property[12].name "rankingExpression(attention_mask).type" +rankprofile[2].fef.property[12].value "tensor<float>(d0[1],d1[128])" +rankprofile[2].fef.property[13].name "rankingExpression(imported_ml_function_tinybert_Mul_6).rankingScript" +rankprofile[2].fef.property[13].value "join(join(1.0, join(join(rename(rankingExpression(attention_mask), (d0, d1), (d0, d4)), tensor<float>(d1[1])(1.0), f(a,b)(a * b)), tensor<float>(d2[1])(1.0), f(a,b)(a * b)), f(a,b)(a - b)), -10000.0, f(a,b)(a * b))" +rankprofile[2].fef.property[14].name "rankingExpression(imported_ml_function_tinybert_Add_88).rankingScript" +rankprofile[2].fef.property[14].value "join(join(reduce(join(tensor<float>(d0[1],d1[12],d2[128],d3[26])((rankingExpression(imported_ml_function_tinybert_Add_47){d0:(0.0), d1:(((39936.0 * d0 + 312.0 * d2 + 26.0 * d1 + d3) % 39936.0) / 312.0), d2:((39936.0 * d0 + 312.0 * d2 + 26.0 * d1 + d3) % 312.0)})), tensor<float>(d0[1],d1[12],d3[26],d4[128])((rankingExpression(imported_ml_function_tinybert_Add_49){d0:(0.0), d1:(((39936.0 * d0 + 312.0 * d4 + 26.0 * d1 + d3) % 39936.0) / 312.0), d2:((39936.0 * d0 + 312.0 * d4 + 26.0 * d1 + d3) % 312.0)})), f(a,b)(a * b)), sum, d3), 5.099019527435303, f(a,b)(a / b)), reduce(rankingExpression(imported_ml_function_tinybert_Mul_6), sum, d1, d2), f(a,b)(a + b))" +rankprofile[2].fef.property[15].name "rankingExpression(imported_ml_function_tinybert_Softmax_89_partial).rankingScript" +rankprofile[2].fef.property[15].value "map(join(rankingExpression(imported_ml_function_tinybert_Add_88), reduce(rankingExpression(imported_ml_function_tinybert_Add_88), max, d4), f(a,b)(a - b)), f(a)(exp(a)))" +rankprofile[2].fef.property[16].name "rankingExpression(imported_ml_function_tinybert_Add_51).rankingScript" +rankprofile[2].fef.property[16].value "join(reduce(join(rankingExpression(imported_ml_function_tinybert_Add_45), constant(tinybert_601), f(a,b)(a * b)), sum, d3), constant(tinybert_encoder_layer_0_attention_self_value_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[17].name "rankingExpression(imported_ml_function_tinybert_Transpose_91).rankingScript" +rankprofile[2].fef.property[17].value "reduce(join(join(rankingExpression(imported_ml_function_tinybert_Softmax_89_partial), reduce(rankingExpression(imported_ml_function_tinybert_Softmax_89_partial), sum, d4), f(a,b)(a / b)), tensor<float>(d0[1],d1[12],d3[26],d4[128])((rankingExpression(imported_ml_function_tinybert_Add_51){d0:(0.0), d1:(((39936.0 * d0 + 312.0 * d4 + 26.0 * d1 + d3) % 39936.0) / 312.0), d2:((39936.0 * d0 + 312.0 * d4 + 26.0 * d1 + d3) % 312.0)})), f(a,b)(a * b)), sum, d4)" +rankprofile[2].fef.property[18].name "rankingExpression(imported_ml_function_tinybert_Add_104).rankingScript" +rankprofile[2].fef.property[18].value "join(join(reduce(join(tensor<float>(d0[1],d1[128],d2[312])((rankingExpression(imported_ml_function_tinybert_Transpose_91){d0:(0.0), d2:(((39936.0 * d0 + 312.0 * d1 + d2) % 39936.0) / 312.0), d1:(((39936.0 * d0 + 312.0 * d1 + d2) % 312.0) / 26.0), d3:((39936.0 * d0 + 312.0 * d1 + d2) % 26.0)})), constant(tinybert_609), f(a,b)(a * b)), sum, d2), constant(tinybert_encoder_layer_0_attention_output_dense_bias), f(a,b)(a + b)), rankingExpression(imported_ml_function_tinybert_Add_45), f(a,b)(a + b))" +rankprofile[2].fef.property[19].name "rankingExpression(imported_ml_function_tinybert_Sub_106).rankingScript" +rankprofile[2].fef.property[19].value "join(rankingExpression(imported_ml_function_tinybert_Add_104), reduce(join(reduce(rankingExpression(imported_ml_function_tinybert_Add_104), avg, d3), tensor<float>(d3[1])(1.0), f(a,b)(a * b)), sum, d3), f(a,b)(a - b))" +rankprofile[2].fef.property[20].name "rankingExpression(imported_ml_function_tinybert_Add_115).rankingScript" +rankprofile[2].fef.property[20].value "join(join(join(rankingExpression(imported_ml_function_tinybert_Sub_106), reduce(map(join(join(reduce(join(rankingExpression(imported_ml_function_tinybert_Sub_106), 2.0, f(a,b)(pow(a,b))), avg, d3), tensor<float>(d3[1])(1.0), f(a,b)(a * b)), 9.999999960041972E-13, f(a,b)(a + b)), f(a)(sqrt(a))), sum, d3), f(a,b)(a / b)), constant(tinybert_encoder_layer_0_attention_output_LayerNorm_weight), f(a,b)(a * b)), constant(tinybert_encoder_layer_0_attention_output_LayerNorm_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[21].name "rankingExpression(imported_ml_function_tinybert_Add_117).rankingScript" +rankprofile[2].fef.property[21].value "join(reduce(join(rankingExpression(imported_ml_function_tinybert_Add_115), constant(tinybert_611), f(a,b)(a * b)), sum, d3), constant(tinybert_encoder_layer_0_intermediate_dense_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[22].name "rankingExpression(imported_ml_function_tinybert_Add_128).rankingScript" +rankprofile[2].fef.property[22].value "join(join(reduce(join(join(join(rankingExpression(imported_ml_function_tinybert_Add_117), join(map(join(rankingExpression(imported_ml_function_tinybert_Add_117), 1.4142135381698608, f(a,b)(a / b)), f(a)(erf(a))), 1.0, f(a,b)(a + b)), f(a,b)(a * b)), 0.5, f(a,b)(a * b)), constant(tinybert_612), f(a,b)(a * b)), sum, d2), constant(tinybert_encoder_layer_0_output_dense_bias), f(a,b)(a + b)), rankingExpression(imported_ml_function_tinybert_Add_115), f(a,b)(a + b))" +rankprofile[2].fef.property[23].name "rankingExpression(imported_ml_function_tinybert_Sub_130).rankingScript" +rankprofile[2].fef.property[23].value "join(rankingExpression(imported_ml_function_tinybert_Add_128), reduce(join(reduce(rankingExpression(imported_ml_function_tinybert_Add_128), avg, d3), tensor<float>(d3[1])(1.0), f(a,b)(a * b)), sum, d3), f(a,b)(a - b))" +rankprofile[2].fef.property[24].name "rankingExpression(imported_ml_function_tinybert_Add_139).rankingScript" +rankprofile[2].fef.property[24].value "join(join(join(rankingExpression(imported_ml_function_tinybert_Sub_130), reduce(map(join(join(reduce(join(rankingExpression(imported_ml_function_tinybert_Sub_130), 2.0, f(a,b)(pow(a,b))), avg, d3), tensor<float>(d3[1])(1.0), f(a,b)(a * b)), 9.999999960041972E-13, f(a,b)(a + b)), f(a)(sqrt(a))), sum, d3), f(a,b)(a / b)), constant(tinybert_encoder_layer_0_output_LayerNorm_weight), f(a,b)(a * b)), constant(tinybert_encoder_layer_0_output_LayerNorm_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[25].name "rankingExpression(imported_ml_function_tinybert_Add_141).rankingScript" +rankprofile[2].fef.property[25].value "join(reduce(join(rankingExpression(imported_ml_function_tinybert_Add_139), constant(tinybert_614), f(a,b)(a * b)), sum, d3), constant(tinybert_encoder_layer_1_attention_self_query_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[26].name "rankingExpression(imported_ml_function_tinybert_Add_143).rankingScript" +rankprofile[2].fef.property[26].value "join(reduce(join(rankingExpression(imported_ml_function_tinybert_Add_139), constant(tinybert_615), f(a,b)(a * b)), sum, d3), constant(tinybert_encoder_layer_1_attention_self_key_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[27].name "rankingExpression(imported_ml_function_tinybert_Add_182).rankingScript" +rankprofile[2].fef.property[27].value "join(join(reduce(join(tensor<float>(d0[1],d1[12],d2[128],d3[26])((rankingExpression(imported_ml_function_tinybert_Add_141){d0:(0.0), d1:(((39936.0 * d0 + 312.0 * d2 + 26.0 * d1 + d3) % 39936.0) / 312.0), d2:((39936.0 * d0 + 312.0 * d2 + 26.0 * d1 + d3) % 312.0)})), tensor<float>(d0[1],d1[12],d3[26],d4[128])((rankingExpression(imported_ml_function_tinybert_Add_143){d0:(0.0), d1:(((39936.0 * d0 + 312.0 * d4 + 26.0 * d1 + d3) % 39936.0) / 312.0), d2:((39936.0 * d0 + 312.0 * d4 + 26.0 * d1 + d3) % 312.0)})), f(a,b)(a * b)), sum, d3), 5.099019527435303, f(a,b)(a / b)), reduce(rankingExpression(imported_ml_function_tinybert_Mul_6), sum, d1, d2), f(a,b)(a + b))" +rankprofile[2].fef.property[28].name "rankingExpression(imported_ml_function_tinybert_Softmax_183_partial).rankingScript" +rankprofile[2].fef.property[28].value "map(join(rankingExpression(imported_ml_function_tinybert_Add_182), reduce(rankingExpression(imported_ml_function_tinybert_Add_182), max, d4), f(a,b)(a - b)), f(a)(exp(a)))" +rankprofile[2].fef.property[29].name "rankingExpression(imported_ml_function_tinybert_Add_145).rankingScript" +rankprofile[2].fef.property[29].value "join(reduce(join(rankingExpression(imported_ml_function_tinybert_Add_139), constant(tinybert_616), f(a,b)(a * b)), sum, d3), constant(tinybert_encoder_layer_1_attention_self_value_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[30].name "rankingExpression(imported_ml_function_tinybert_Transpose_185).rankingScript" +rankprofile[2].fef.property[30].value "reduce(join(join(rankingExpression(imported_ml_function_tinybert_Softmax_183_partial), reduce(rankingExpression(imported_ml_function_tinybert_Softmax_183_partial), sum, d4), f(a,b)(a / b)), tensor<float>(d0[1],d1[12],d3[26],d4[128])((rankingExpression(imported_ml_function_tinybert_Add_145){d0:(0.0), d1:(((39936.0 * d0 + 312.0 * d4 + 26.0 * d1 + d3) % 39936.0) / 312.0), d2:((39936.0 * d0 + 312.0 * d4 + 26.0 * d1 + d3) % 312.0)})), f(a,b)(a * b)), sum, d4)" +rankprofile[2].fef.property[31].name "rankingExpression(imported_ml_function_tinybert_Add_198).rankingScript" +rankprofile[2].fef.property[31].value "join(join(reduce(join(tensor<float>(d0[1],d1[128],d2[312])((rankingExpression(imported_ml_function_tinybert_Transpose_185){d0:(0.0), d2:(((39936.0 * d0 + 312.0 * d1 + d2) % 39936.0) / 312.0), d1:(((39936.0 * d0 + 312.0 * d1 + d2) % 312.0) / 26.0), d3:((39936.0 * d0 + 312.0 * d1 + d2) % 26.0)})), constant(tinybert_624), f(a,b)(a * b)), sum, d2), constant(tinybert_encoder_layer_1_attention_output_dense_bias), f(a,b)(a + b)), rankingExpression(imported_ml_function_tinybert_Add_139), f(a,b)(a + b))" +rankprofile[2].fef.property[32].name "rankingExpression(imported_ml_function_tinybert_Sub_200).rankingScript" +rankprofile[2].fef.property[32].value "join(rankingExpression(imported_ml_function_tinybert_Add_198), reduce(join(reduce(rankingExpression(imported_ml_function_tinybert_Add_198), avg, d3), tensor<float>(d3[1])(1.0), f(a,b)(a * b)), sum, d3), f(a,b)(a - b))" +rankprofile[2].fef.property[33].name "rankingExpression(imported_ml_function_tinybert_Add_209).rankingScript" +rankprofile[2].fef.property[33].value "join(join(join(rankingExpression(imported_ml_function_tinybert_Sub_200), reduce(map(join(join(reduce(join(rankingExpression(imported_ml_function_tinybert_Sub_200), 2.0, f(a,b)(pow(a,b))), avg, d3), tensor<float>(d3[1])(1.0), f(a,b)(a * b)), 9.999999960041972E-13, f(a,b)(a + b)), f(a)(sqrt(a))), sum, d3), f(a,b)(a / b)), constant(tinybert_encoder_layer_1_attention_output_LayerNorm_weight), f(a,b)(a * b)), constant(tinybert_encoder_layer_1_attention_output_LayerNorm_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[34].name "rankingExpression(imported_ml_function_tinybert_Add_211).rankingScript" +rankprofile[2].fef.property[34].value "join(reduce(join(rankingExpression(imported_ml_function_tinybert_Add_209), constant(tinybert_626), f(a,b)(a * b)), sum, d3), constant(tinybert_encoder_layer_1_intermediate_dense_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[35].name "rankingExpression(imported_ml_function_tinybert_Add_222).rankingScript" +rankprofile[2].fef.property[35].value "join(join(reduce(join(join(join(rankingExpression(imported_ml_function_tinybert_Add_211), join(map(join(rankingExpression(imported_ml_function_tinybert_Add_211), 1.4142135381698608, f(a,b)(a / b)), f(a)(erf(a))), 1.0, f(a,b)(a + b)), f(a,b)(a * b)), 0.5, f(a,b)(a * b)), constant(tinybert_627), f(a,b)(a * b)), sum, d2), constant(tinybert_encoder_layer_1_output_dense_bias), f(a,b)(a + b)), rankingExpression(imported_ml_function_tinybert_Add_209), f(a,b)(a + b))" +rankprofile[2].fef.property[36].name "rankingExpression(imported_ml_function_tinybert_Sub_224).rankingScript" +rankprofile[2].fef.property[36].value "join(rankingExpression(imported_ml_function_tinybert_Add_222), reduce(join(reduce(rankingExpression(imported_ml_function_tinybert_Add_222), avg, d3), tensor<float>(d3[1])(1.0), f(a,b)(a * b)), sum, d3), f(a,b)(a - b))" +rankprofile[2].fef.property[37].name "rankingExpression(imported_ml_function_tinybert_Add_233).rankingScript" +rankprofile[2].fef.property[37].value "join(join(join(rankingExpression(imported_ml_function_tinybert_Sub_224), reduce(map(join(join(reduce(join(rankingExpression(imported_ml_function_tinybert_Sub_224), 2.0, f(a,b)(pow(a,b))), avg, d3), tensor<float>(d3[1])(1.0), f(a,b)(a * b)), 9.999999960041972E-13, f(a,b)(a + b)), f(a)(sqrt(a))), sum, d3), f(a,b)(a / b)), constant(tinybert_encoder_layer_1_output_LayerNorm_weight), f(a,b)(a * b)), constant(tinybert_encoder_layer_1_output_LayerNorm_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[38].name "rankingExpression(imported_ml_function_tinybert_Add_235).rankingScript" +rankprofile[2].fef.property[38].value "join(reduce(join(rankingExpression(imported_ml_function_tinybert_Add_233), constant(tinybert_629), f(a,b)(a * b)), sum, d3), constant(tinybert_encoder_layer_2_attention_self_query_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[39].name "rankingExpression(imported_ml_function_tinybert_Add_237).rankingScript" +rankprofile[2].fef.property[39].value "join(reduce(join(rankingExpression(imported_ml_function_tinybert_Add_233), constant(tinybert_630), f(a,b)(a * b)), sum, d3), constant(tinybert_encoder_layer_2_attention_self_key_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[40].name "rankingExpression(imported_ml_function_tinybert_Add_276).rankingScript" +rankprofile[2].fef.property[40].value "join(join(reduce(join(tensor<float>(d0[1],d1[12],d2[128],d3[26])((rankingExpression(imported_ml_function_tinybert_Add_235){d0:(0.0), d1:(((39936.0 * d0 + 312.0 * d2 + 26.0 * d1 + d3) % 39936.0) / 312.0), d2:((39936.0 * d0 + 312.0 * d2 + 26.0 * d1 + d3) % 312.0)})), tensor<float>(d0[1],d1[12],d3[26],d4[128])((rankingExpression(imported_ml_function_tinybert_Add_237){d0:(0.0), d1:(((39936.0 * d0 + 312.0 * d4 + 26.0 * d1 + d3) % 39936.0) / 312.0), d2:((39936.0 * d0 + 312.0 * d4 + 26.0 * d1 + d3) % 312.0)})), f(a,b)(a * b)), sum, d3), 5.099019527435303, f(a,b)(a / b)), reduce(rankingExpression(imported_ml_function_tinybert_Mul_6), sum, d1, d2), f(a,b)(a + b))" +rankprofile[2].fef.property[41].name "rankingExpression(imported_ml_function_tinybert_Softmax_277_partial).rankingScript" +rankprofile[2].fef.property[41].value "map(join(rankingExpression(imported_ml_function_tinybert_Add_276), reduce(rankingExpression(imported_ml_function_tinybert_Add_276), max, d4), f(a,b)(a - b)), f(a)(exp(a)))" +rankprofile[2].fef.property[42].name "rankingExpression(imported_ml_function_tinybert_Add_239).rankingScript" +rankprofile[2].fef.property[42].value "join(reduce(join(rankingExpression(imported_ml_function_tinybert_Add_233), constant(tinybert_631), f(a,b)(a * b)), sum, d3), constant(tinybert_encoder_layer_2_attention_self_value_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[43].name "rankingExpression(imported_ml_function_tinybert_Transpose_279).rankingScript" +rankprofile[2].fef.property[43].value "reduce(join(join(rankingExpression(imported_ml_function_tinybert_Softmax_277_partial), reduce(rankingExpression(imported_ml_function_tinybert_Softmax_277_partial), sum, d4), f(a,b)(a / b)), tensor<float>(d0[1],d1[12],d3[26],d4[128])((rankingExpression(imported_ml_function_tinybert_Add_239){d0:(0.0), d1:(((39936.0 * d0 + 312.0 * d4 + 26.0 * d1 + d3) % 39936.0) / 312.0), d2:((39936.0 * d0 + 312.0 * d4 + 26.0 * d1 + d3) % 312.0)})), f(a,b)(a * b)), sum, d4)" +rankprofile[2].fef.property[44].name "rankingExpression(imported_ml_function_tinybert_Add_292).rankingScript" +rankprofile[2].fef.property[44].value "join(join(reduce(join(tensor<float>(d0[1],d1[128],d2[312])((rankingExpression(imported_ml_function_tinybert_Transpose_279){d0:(0.0), d2:(((39936.0 * d0 + 312.0 * d1 + d2) % 39936.0) / 312.0), d1:(((39936.0 * d0 + 312.0 * d1 + d2) % 312.0) / 26.0), d3:((39936.0 * d0 + 312.0 * d1 + d2) % 26.0)})), constant(tinybert_639), f(a,b)(a * b)), sum, d2), constant(tinybert_encoder_layer_2_attention_output_dense_bias), f(a,b)(a + b)), rankingExpression(imported_ml_function_tinybert_Add_233), f(a,b)(a + b))" +rankprofile[2].fef.property[45].name "rankingExpression(imported_ml_function_tinybert_Sub_294).rankingScript" +rankprofile[2].fef.property[45].value "join(rankingExpression(imported_ml_function_tinybert_Add_292), reduce(join(reduce(rankingExpression(imported_ml_function_tinybert_Add_292), avg, d3), tensor<float>(d3[1])(1.0), f(a,b)(a * b)), sum, d3), f(a,b)(a - b))" +rankprofile[2].fef.property[46].name "rankingExpression(imported_ml_function_tinybert_Add_303).rankingScript" +rankprofile[2].fef.property[46].value "join(join(join(rankingExpression(imported_ml_function_tinybert_Sub_294), reduce(map(join(join(reduce(join(rankingExpression(imported_ml_function_tinybert_Sub_294), 2.0, f(a,b)(pow(a,b))), avg, d3), tensor<float>(d3[1])(1.0), f(a,b)(a * b)), 9.999999960041972E-13, f(a,b)(a + b)), f(a)(sqrt(a))), sum, d3), f(a,b)(a / b)), constant(tinybert_encoder_layer_2_attention_output_LayerNorm_weight), f(a,b)(a * b)), constant(tinybert_encoder_layer_2_attention_output_LayerNorm_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[47].name "rankingExpression(imported_ml_function_tinybert_Add_305).rankingScript" +rankprofile[2].fef.property[47].value "join(reduce(join(rankingExpression(imported_ml_function_tinybert_Add_303), constant(tinybert_641), f(a,b)(a * b)), sum, d3), constant(tinybert_encoder_layer_2_intermediate_dense_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[48].name "rankingExpression(imported_ml_function_tinybert_Add_316).rankingScript" +rankprofile[2].fef.property[48].value "join(join(reduce(join(join(join(rankingExpression(imported_ml_function_tinybert_Add_305), join(map(join(rankingExpression(imported_ml_function_tinybert_Add_305), 1.4142135381698608, f(a,b)(a / b)), f(a)(erf(a))), 1.0, f(a,b)(a + b)), f(a,b)(a * b)), 0.5, f(a,b)(a * b)), constant(tinybert_642), f(a,b)(a * b)), sum, d2), constant(tinybert_encoder_layer_2_output_dense_bias), f(a,b)(a + b)), rankingExpression(imported_ml_function_tinybert_Add_303), f(a,b)(a + b))" +rankprofile[2].fef.property[49].name "rankingExpression(imported_ml_function_tinybert_Sub_318).rankingScript" +rankprofile[2].fef.property[49].value "join(rankingExpression(imported_ml_function_tinybert_Add_316), reduce(join(reduce(rankingExpression(imported_ml_function_tinybert_Add_316), avg, d3), tensor<float>(d3[1])(1.0), f(a,b)(a * b)), sum, d3), f(a,b)(a - b))" +rankprofile[2].fef.property[50].name "rankingExpression(imported_ml_function_tinybert_Add_327).rankingScript" +rankprofile[2].fef.property[50].value "join(join(join(rankingExpression(imported_ml_function_tinybert_Sub_318), reduce(map(join(join(reduce(join(rankingExpression(imported_ml_function_tinybert_Sub_318), 2.0, f(a,b)(pow(a,b))), avg, d3), tensor<float>(d3[1])(1.0), f(a,b)(a * b)), 9.999999960041972E-13, f(a,b)(a + b)), f(a)(sqrt(a))), sum, d3), f(a,b)(a / b)), constant(tinybert_encoder_layer_2_output_LayerNorm_weight), f(a,b)(a * b)), constant(tinybert_encoder_layer_2_output_LayerNorm_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[51].name "rankingExpression(imported_ml_function_tinybert_Add_329).rankingScript" +rankprofile[2].fef.property[51].value "join(reduce(join(rankingExpression(imported_ml_function_tinybert_Add_327), constant(tinybert_644), f(a,b)(a * b)), sum, d3), constant(tinybert_encoder_layer_3_attention_self_query_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[52].name "rankingExpression(imported_ml_function_tinybert_Add_331).rankingScript" +rankprofile[2].fef.property[52].value "join(reduce(join(rankingExpression(imported_ml_function_tinybert_Add_327), constant(tinybert_645), f(a,b)(a * b)), sum, d3), constant(tinybert_encoder_layer_3_attention_self_key_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[53].name "rankingExpression(imported_ml_function_tinybert_Add_370).rankingScript" +rankprofile[2].fef.property[53].value "join(join(reduce(join(tensor<float>(d0[1],d1[12],d2[128],d3[26])((rankingExpression(imported_ml_function_tinybert_Add_329){d0:(0.0), d1:(((39936.0 * d0 + 312.0 * d2 + 26.0 * d1 + d3) % 39936.0) / 312.0), d2:((39936.0 * d0 + 312.0 * d2 + 26.0 * d1 + d3) % 312.0)})), tensor<float>(d0[1],d1[12],d3[26],d4[128])((rankingExpression(imported_ml_function_tinybert_Add_331){d0:(0.0), d1:(((39936.0 * d0 + 312.0 * d4 + 26.0 * d1 + d3) % 39936.0) / 312.0), d2:((39936.0 * d0 + 312.0 * d4 + 26.0 * d1 + d3) % 312.0)})), f(a,b)(a * b)), sum, d3), 5.099019527435303, f(a,b)(a / b)), reduce(rankingExpression(imported_ml_function_tinybert_Mul_6), sum, d1, d2), f(a,b)(a + b))" +rankprofile[2].fef.property[54].name "rankingExpression(imported_ml_function_tinybert_Softmax_371_partial).rankingScript" +rankprofile[2].fef.property[54].value "map(join(rankingExpression(imported_ml_function_tinybert_Add_370), reduce(rankingExpression(imported_ml_function_tinybert_Add_370), max, d4), f(a,b)(a - b)), f(a)(exp(a)))" +rankprofile[2].fef.property[55].name "rankingExpression(imported_ml_function_tinybert_Add_333).rankingScript" +rankprofile[2].fef.property[55].value "join(reduce(join(rankingExpression(imported_ml_function_tinybert_Add_327), constant(tinybert_646), f(a,b)(a * b)), sum, d3), constant(tinybert_encoder_layer_3_attention_self_value_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[56].name "rankingExpression(imported_ml_function_tinybert_Transpose_373).rankingScript" +rankprofile[2].fef.property[56].value "reduce(join(join(rankingExpression(imported_ml_function_tinybert_Softmax_371_partial), reduce(rankingExpression(imported_ml_function_tinybert_Softmax_371_partial), sum, d4), f(a,b)(a / b)), tensor<float>(d0[1],d1[12],d3[26],d4[128])((rankingExpression(imported_ml_function_tinybert_Add_333){d0:(0.0), d1:(((39936.0 * d0 + 312.0 * d4 + 26.0 * d1 + d3) % 39936.0) / 312.0), d2:((39936.0 * d0 + 312.0 * d4 + 26.0 * d1 + d3) % 312.0)})), f(a,b)(a * b)), sum, d4)" +rankprofile[2].fef.property[57].name "rankingExpression(imported_ml_function_tinybert_Add_386).rankingScript" +rankprofile[2].fef.property[57].value "join(join(reduce(join(tensor<float>(d0[1],d1[128],d2[312])((rankingExpression(imported_ml_function_tinybert_Transpose_373){d0:(0.0), d2:(((39936.0 * d0 + 312.0 * d1 + d2) % 39936.0) / 312.0), d1:(((39936.0 * d0 + 312.0 * d1 + d2) % 312.0) / 26.0), d3:((39936.0 * d0 + 312.0 * d1 + d2) % 26.0)})), constant(tinybert_654), f(a,b)(a * b)), sum, d2), constant(tinybert_encoder_layer_3_attention_output_dense_bias), f(a,b)(a + b)), rankingExpression(imported_ml_function_tinybert_Add_327), f(a,b)(a + b))" +rankprofile[2].fef.property[58].name "rankingExpression(imported_ml_function_tinybert_Sub_388).rankingScript" +rankprofile[2].fef.property[58].value "join(rankingExpression(imported_ml_function_tinybert_Add_386), reduce(join(reduce(rankingExpression(imported_ml_function_tinybert_Add_386), avg, d3), tensor<float>(d3[1])(1.0), f(a,b)(a * b)), sum, d3), f(a,b)(a - b))" +rankprofile[2].fef.property[59].name "rankingExpression(imported_ml_function_tinybert_Add_397).rankingScript" +rankprofile[2].fef.property[59].value "join(join(join(rankingExpression(imported_ml_function_tinybert_Sub_388), reduce(map(join(join(reduce(join(rankingExpression(imported_ml_function_tinybert_Sub_388), 2.0, f(a,b)(pow(a,b))), avg, d3), tensor<float>(d3[1])(1.0), f(a,b)(a * b)), 9.999999960041972E-13, f(a,b)(a + b)), f(a)(sqrt(a))), sum, d3), f(a,b)(a / b)), constant(tinybert_encoder_layer_3_attention_output_LayerNorm_weight), f(a,b)(a * b)), constant(tinybert_encoder_layer_3_attention_output_LayerNorm_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[60].name "rankingExpression(imported_ml_function_tinybert_Add_399).rankingScript" +rankprofile[2].fef.property[60].value "join(reduce(join(rankingExpression(imported_ml_function_tinybert_Add_397), constant(tinybert_656), f(a,b)(a * b)), sum, d3), constant(tinybert_encoder_layer_3_intermediate_dense_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[61].name "rankingExpression(imported_ml_function_tinybert_Add_410).rankingScript" +rankprofile[2].fef.property[61].value "join(join(reduce(join(join(join(rankingExpression(imported_ml_function_tinybert_Add_399), join(map(join(rankingExpression(imported_ml_function_tinybert_Add_399), 1.4142135381698608, f(a,b)(a / b)), f(a)(erf(a))), 1.0, f(a,b)(a + b)), f(a,b)(a * b)), 0.5, f(a,b)(a * b)), constant(tinybert_657), f(a,b)(a * b)), sum, d2), constant(tinybert_encoder_layer_3_output_dense_bias), f(a,b)(a + b)), rankingExpression(imported_ml_function_tinybert_Add_397), f(a,b)(a + b))" +rankprofile[2].fef.property[62].name "rankingExpression(imported_ml_function_tinybert_Sub_412).rankingScript" +rankprofile[2].fef.property[62].value "join(rankingExpression(imported_ml_function_tinybert_Add_410), reduce(join(reduce(rankingExpression(imported_ml_function_tinybert_Add_410), avg, d3), tensor<float>(d3[1])(1.0), f(a,b)(a * b)), sum, d3), f(a,b)(a - b))" +rankprofile[2].fef.property[63].name "rankingExpression(imported_ml_function_tinybert_Add_421).rankingScript" +rankprofile[2].fef.property[63].value "join(join(join(rankingExpression(imported_ml_function_tinybert_Sub_412), reduce(map(join(join(reduce(join(rankingExpression(imported_ml_function_tinybert_Sub_412), 2.0, f(a,b)(pow(a,b))), avg, d3), tensor<float>(d3[1])(1.0), f(a,b)(a * b)), 9.999999960041972E-13, f(a,b)(a + b)), f(a)(sqrt(a))), sum, d3), f(a,b)(a / b)), constant(tinybert_encoder_layer_3_output_LayerNorm_weight), f(a,b)(a * b)), constant(tinybert_encoder_layer_3_output_LayerNorm_bias), f(a,b)(a + b))" +rankprofile[2].fef.property[64].name "vespa.rank.firstphase" +rankprofile[2].fef.property[64].value "rankingExpression(firstphase)" +rankprofile[2].fef.property[65].name "rankingExpression(firstphase).rankingScript" +rankprofile[2].fef.property[65].value "reduce(rename(rankingExpression(imported_ml_function_tinybert_Add_421), (d0, d1, d3), (d0, d1, d2)), sum)" +rankprofile[2].fef.property[66].name "vespa.type.attribute.field_token_type_ids" +rankprofile[2].fef.property[66].value "tensor<float>(d0[1],d1[128])" +rankprofile[2].fef.property[67].name "vespa.type.attribute.field_attention_mask" +rankprofile[2].fef.property[67].value "tensor<float>(d0[1],d1[128])" +rankprofile[2].fef.property[68].name "vespa.type.attribute.field_input_ids" +rankprofile[2].fef.property[68].value "tensor<float>(d0[1],d1[128])" diff --git a/model-evaluation/src/test/resources/config/tinybert/ranking-constants.cfg b/model-evaluation/src/test/resources/config/tinybert/ranking-constants.cfg new file mode 100644 index 00000000000..a35f282f54d --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/ranking-constants.cfg @@ -0,0 +1,261 @@ +constant[0].name "tinybert_encoder_layer_1_output_LayerNorm_weight" +constant[0].fileref "filedistribution/27e2afaa08b38e1/tinybert_encoder_layer_1_output_LayerNorm_weight.tbf" +constant[0].type "tensor<float>(d3[312])" +constant[1].name "tinybert_encoder_layer_2_intermediate_dense_bias" +constant[1].fileref "filedistribution/8fef598bac28930b/tinybert_encoder_layer_2_intermediate_dense_bias.tbf" +constant[1].type "tensor<float>(d2[1200])" +constant[2].name "tinybert_627" +constant[2].fileref "filedistribution/9d8e9504a5b2d19b/tinybert_627.tbf" +constant[2].type "tensor<float>(d2[1200],d3[312])" +constant[3].name "tinybert_pooler_dense_bias" +constant[3].fileref "filedistribution/16bc5a7698891f15/tinybert_pooler_dense_bias.tbf" +constant[3].type "tensor<float>(d1[312])" +constant[4].name "tinybert_encoder_layer_1_attention_self_key_bias" +constant[4].fileref "filedistribution/767334a7ff3e73e7/tinybert_encoder_layer_1_attention_self_key_bias.tbf" +constant[4].type "tensor<float>(d2[312])" +constant[5].name "tinybert_encoder_layer_1_output_dense_bias" +constant[5].fileref "filedistribution/bac9a3d7f3f7e173/tinybert_encoder_layer_1_output_dense_bias.tbf" +constant[5].type "tensor<float>(d3[312])" +constant[6].name "tinybert_encoder_layer_1_intermediate_dense_bias" +constant[6].fileref "filedistribution/ea71f78349b945f3/tinybert_encoder_layer_1_intermediate_dense_bias.tbf" +constant[6].type "tensor<float>(d2[1200])" +constant[7].name "tinybert_encoder_layer_2_output_LayerNorm_weight" +constant[7].fileref "filedistribution/71379a41f448366f/tinybert_encoder_layer_2_output_LayerNorm_weight.tbf" +constant[7].type "tensor<float>(d3[312])" +constant[8].name "tinybert_Concat_363" +constant[8].fileref "filedistribution/1bfb351718ce25f2/tinybert_Concat_363.tbf" +constant[8].type "tensor<float>(d0[4])" +constant[9].name "tinybert_630" +constant[9].fileref "filedistribution/5f4e10876d3b273b/tinybert_630.tbf" +constant[9].type "tensor<float>(d2[312],d3[312])" +constant[10].name "tinybert_Concat_60" +constant[10].fileref "filedistribution/880243d76119d1be/tinybert_Concat_60.tbf" +constant[10].type "tensor<float>(d0[4])" +constant[11].name "tinybert_Concat_259" +constant[11].fileref "filedistribution/669c970d89cc2ba7/tinybert_Concat_259.tbf" +constant[11].type "tensor<float>(d0[4])" +constant[12].name "tinybert_encoder_layer_2_attention_self_query_bias" +constant[12].fileref "filedistribution/a642396691f08a5f/tinybert_encoder_layer_2_attention_self_query_bias.tbf" +constant[12].type "tensor<float>(d2[312])" +constant[13].name "tinybert_encoder_layer_0_output_LayerNorm_bias" +constant[13].fileref "filedistribution/3a3d747f48b9d8c7/tinybert_encoder_layer_0_output_LayerNorm_bias.tbf" +constant[13].type "tensor<float>(d3[312])" +constant[14].name "tinybert_644" +constant[14].fileref "filedistribution/82f078dd5493eab/tinybert_644.tbf" +constant[14].type "tensor<float>(d2[312],d3[312])" +constant[15].name "tinybert_embeddings_LayerNorm_weight" +constant[15].fileref "filedistribution/14b40ffc665653da/tinybert_embeddings_LayerNorm_weight.tbf" +constant[15].type "tensor<float>(d3[312])" +constant[16].name "tinybert_encoder_layer_0_output_LayerNorm_weight" +constant[16].fileref "filedistribution/31795fa4ab9cdcee/tinybert_encoder_layer_0_output_LayerNorm_weight.tbf" +constant[16].type "tensor<float>(d3[312])" +constant[17].name "tinybert_encoder_layer_0_intermediate_dense_bias" +constant[17].fileref "filedistribution/9ad03013d8ea199b/tinybert_encoder_layer_0_intermediate_dense_bias.tbf" +constant[17].type "tensor<float>(d2[1200])" +constant[18].name "tinybert_encoder_layer_3_attention_self_value_bias" +constant[18].fileref "filedistribution/a10cda99920a91b7/tinybert_encoder_layer_3_attention_self_value_bias.tbf" +constant[18].type "tensor<float>(d2[312])" +constant[19].name "tinybert_encoder_layer_1_attention_output_LayerNorm_bias" +constant[19].fileref "filedistribution/c6c6f63e836f2582/tinybert_encoder_layer_1_attention_output_LayerNorm_bias.tbf" +constant[19].type "tensor<float>(d3[312])" +constant[20].name "tinybert_encoder_layer_0_attention_self_key_bias" +constant[20].fileref "filedistribution/61115ebdbd912cc1/tinybert_encoder_layer_0_attention_self_key_bias.tbf" +constant[20].type "tensor<float>(d2[312])" +constant[21].name "tinybert_Concat_194" +constant[21].fileref "filedistribution/bb406a628e9df4d3/tinybert_Concat_194.tbf" +constant[21].type "tensor<float>(d0[3])" +constant[22].name "tinybert_609" +constant[22].fileref "filedistribution/e86d7b0aeaea757c/tinybert_609.tbf" +constant[22].type "tensor<float>(d2[312],d3[312])" +constant[23].name "tinybert_pooler_dense_weight" +constant[23].fileref "filedistribution/a05168f0aef055b4/tinybert_pooler_dense_weight.tbf" +constant[23].type "tensor<float>(d1[312],d3[312])" +constant[24].name "tinybert_642" +constant[24].fileref "filedistribution/fc60417d2ee421df/tinybert_642.tbf" +constant[24].type "tensor<float>(d2[1200],d3[312])" +constant[25].name "tinybert_encoder_layer_1_attention_self_query_bias" +constant[25].fileref "filedistribution/704d97f73a6ea9fb/tinybert_encoder_layer_1_attention_self_query_bias.tbf" +constant[25].type "tensor<float>(d2[312])" +constant[26].name "tinybert_Concat_175" +constant[26].fileref "filedistribution/6b9c014f349b42b3/tinybert_Concat_175.tbf" +constant[26].type "tensor<float>(d0[4])" +constant[27].name "tinybert_embeddings_LayerNorm_bias" +constant[27].fileref "filedistribution/590fe4d33400f007/tinybert_embeddings_LayerNorm_bias.tbf" +constant[27].type "tensor<float>(d3[312])" +constant[28].name "tinybert_656" +constant[28].fileref "filedistribution/c2195a93d2c7aa5c/tinybert_656.tbf" +constant[28].type "tensor<float>(d2[1200],d3[312])" +constant[29].name "tinybert_631" +constant[29].fileref "filedistribution/62013069bfe7e037/tinybert_631.tbf" +constant[29].type "tensor<float>(d2[312],d3[312])" +constant[30].name "tinybert_encoder_layer_2_attention_self_key_bias" +constant[30].fileref "filedistribution/69bd5a278268d6a8/tinybert_encoder_layer_2_attention_self_key_bias.tbf" +constant[30].type "tensor<float>(d2[312])" +constant[31].name "tinybert_encoder_layer_2_output_dense_bias" +constant[31].fileref "filedistribution/d25674a64681be9f/tinybert_encoder_layer_2_output_dense_bias.tbf" +constant[31].type "tensor<float>(d3[312])" +constant[32].name "tinybert_encoder_layer_1_attention_output_dense_bias" +constant[32].fileref "filedistribution/1e0c4bc8a082d938/tinybert_encoder_layer_1_attention_output_dense_bias.tbf" +constant[32].type "tensor<float>(d3[312])" +constant[33].name "tinybert_645" +constant[33].fileref "filedistribution/7ef6de3dd8902534/tinybert_645.tbf" +constant[33].type "tensor<float>(d2[312],d3[312])" +constant[34].name "tinybert_encoder_layer_3_attention_output_dense_bias" +constant[34].fileref "filedistribution/9a8e585e32950ecf/tinybert_encoder_layer_3_attention_output_dense_bias.tbf" +constant[34].type "tensor<float>(d3[312])" +constant[35].name "tinybert_Concat_353" +constant[35].fileref "filedistribution/dc72a0e790a30020/tinybert_Concat_353.tbf" +constant[35].type "tensor<float>(d0[4])" +constant[36].name "tinybert_Concat_342" +constant[36].fileref "filedistribution/db6dae3a39fcf192/tinybert_Concat_342.tbf" +constant[36].type "tensor<float>(d0[4])" +constant[37].name "tinybert_626" +constant[37].fileref "filedistribution/c165d1ea133a28b6/tinybert_626.tbf" +constant[37].type "tensor<float>(d2[1200],d3[312])" +constant[38].name "tinybert_612" +constant[38].fileref "filedistribution/f0a05a2cea97e1f9/tinybert_612.tbf" +constant[38].type "tensor<float>(d2[1200],d3[312])" +constant[39].name "tinybert_embeddings_token_type_embeddings_weight" +constant[39].fileref "filedistribution/7d01968ada002ff8/tinybert_embeddings_token_type_embeddings_weight.tbf" +constant[39].type "tensor<float>(d0[2],d3[312])" +constant[40].name "tinybert_encoder_layer_3_intermediate_dense_bias" +constant[40].fileref "filedistribution/a893d4ac541d42eb/tinybert_encoder_layer_3_intermediate_dense_bias.tbf" +constant[40].type "tensor<float>(d2[1200])" +constant[41].name "tinybert_encoder_layer_0_attention_self_query_bias" +constant[41].fileref "filedistribution/5bdfe53fc08f095/tinybert_encoder_layer_0_attention_self_query_bias.tbf" +constant[41].type "tensor<float>(d2[312])" +constant[42].name "tinybert_615" +constant[42].fileref "filedistribution/1667a5cf592c365/tinybert_615.tbf" +constant[42].type "tensor<float>(d2[312],d3[312])" +constant[43].name "tinybert_encoder_layer_2_output_LayerNorm_bias" +constant[43].fileref "filedistribution/ccd015582d66388e/tinybert_encoder_layer_2_output_LayerNorm_bias.tbf" +constant[43].type "tensor<float>(d3[312])" +constant[44].name "tinybert_601" +constant[44].fileref "filedistribution/c003ee0fce20531/tinybert_601.tbf" +constant[44].type "tensor<float>(d2[312],d3[312])" +constant[45].name "tinybert_Concat_81" +constant[45].fileref "filedistribution/cd0d517153ca69b4/tinybert_Concat_81.tbf" +constant[45].type "tensor<float>(d0[4])" +constant[46].name "tinybert_624" +constant[46].fileref "filedistribution/52e4f13729329216/tinybert_624.tbf" +constant[46].type "tensor<float>(d2[312],d3[312])" +constant[47].name "tinybert_Concat_288" +constant[47].fileref "filedistribution/8216d52cc5aa31bd/tinybert_Concat_288.tbf" +constant[47].type "tensor<float>(d0[3])" +constant[48].name "tinybert_Gather_31" +constant[48].fileref "filedistribution/a25a5243e3c26732/tinybert_Gather_31.tbf" +constant[48].type "tensor<float>(d0[1],d1[128],d3[312])" +constant[49].name "tinybert_encoder_layer_3_output_LayerNorm_bias" +constant[49].fileref "filedistribution/13cb87130508f381/tinybert_encoder_layer_3_output_LayerNorm_bias.tbf" +constant[49].type "tensor<float>(d3[312])" +constant[50].name "tinybert_encoder_layer_3_attention_self_query_bias" +constant[50].fileref "filedistribution/261b74d9b1faa77d/tinybert_encoder_layer_3_attention_self_query_bias.tbf" +constant[50].type "tensor<float>(d2[312])" +constant[51].name "tinybert_616" +constant[51].fileref "filedistribution/5d4350323a071e9f/tinybert_616.tbf" +constant[51].type "tensor<float>(d2[312],d3[312])" +constant[52].name "tinybert_Concat_382" +constant[52].fileref "filedistribution/7c076b59f45223ef/tinybert_Concat_382.tbf" +constant[52].type "tensor<float>(d0[3])" +constant[53].name "tinybert_encoder_layer_3_output_dense_bias" +constant[53].fileref "filedistribution/c0374bdd83f6f831/tinybert_encoder_layer_3_output_dense_bias.tbf" +constant[53].type "tensor<float>(d3[312])" +constant[54].name "tinybert_encoder_layer_0_attention_output_LayerNorm_bias" +constant[54].fileref "filedistribution/1bdfde9445c7cb6d/tinybert_encoder_layer_0_attention_output_LayerNorm_bias.tbf" +constant[54].type "tensor<float>(d3[312])" +constant[55].name "tinybert_encoder_layer_2_attention_self_value_bias" +constant[55].fileref "filedistribution/93736ee6488d12f9/tinybert_encoder_layer_2_attention_self_value_bias.tbf" +constant[55].type "tensor<float>(d2[312])" +constant[56].name "tinybert_629" +constant[56].fileref "filedistribution/5571e3625094b475/tinybert_629.tbf" +constant[56].type "tensor<float>(d2[312],d3[312])" +constant[57].name "tinybert_encoder_layer_2_attention_output_LayerNorm_bias" +constant[57].fileref "filedistribution/5326aaa30d3cd2fd/tinybert_encoder_layer_2_attention_output_LayerNorm_bias.tbf" +constant[57].type "tensor<float>(d3[312])" +constant[58].name "tinybert_Concat_248" +constant[58].fileref "filedistribution/60bab4fc23fe183e/tinybert_Concat_248.tbf" +constant[58].type "tensor<float>(d0[4])" +constant[59].name "tinybert_639" +constant[59].fileref "filedistribution/f063f0b40260827d/tinybert_639.tbf" +constant[59].type "tensor<float>(d2[312],d3[312])" +constant[60].name "tinybert_Concat_71" +constant[60].fileref "filedistribution/dfaa3c067cc9a6ff/tinybert_Concat_71.tbf" +constant[60].type "tensor<float>(d0[4])" +constant[61].name "tinybert_641" +constant[61].fileref "filedistribution/a7d13f6c41a99226/tinybert_641.tbf" +constant[61].type "tensor<float>(d2[1200],d3[312])" +constant[62].name "tinybert_600" +constant[62].fileref "filedistribution/36b4a3d779ef7201/tinybert_600.tbf" +constant[62].type "tensor<float>(d2[312],d3[312])" +constant[63].name "tinybert_encoder_layer_3_attention_output_LayerNorm_bias" +constant[63].fileref "filedistribution/12c4ee4c5547a64e/tinybert_encoder_layer_3_attention_output_LayerNorm_bias.tbf" +constant[63].type "tensor<float>(d3[312])" +constant[64].name "tinybert_614" +constant[64].fileref "filedistribution/689165a57d7656c9/tinybert_614.tbf" +constant[64].type "tensor<float>(d2[312],d3[312])" +constant[65].name "tinybert_611" +constant[65].fileref "filedistribution/2901f3bcfdb170a4/tinybert_611.tbf" +constant[65].type "tensor<float>(d2[1200],d3[312])" +constant[66].name "tinybert_encoder_layer_2_attention_output_dense_bias" +constant[66].fileref "filedistribution/99f6d1ba780ffe21/tinybert_encoder_layer_2_attention_output_dense_bias.tbf" +constant[66].type "tensor<float>(d3[312])" +constant[67].name "tinybert_encoder_layer_1_output_LayerNorm_bias" +constant[67].fileref "filedistribution/c5e300a8e998ce7f/tinybert_encoder_layer_1_output_LayerNorm_bias.tbf" +constant[67].type "tensor<float>(d3[312])" +constant[68].name "tinybert_encoder_layer_0_attention_output_dense_bias" +constant[68].fileref "filedistribution/ac91e41746178779/tinybert_encoder_layer_0_attention_output_dense_bias.tbf" +constant[68].type "tensor<float>(d3[312])" +constant[69].name "tinybert_encoder_layer_0_output_dense_bias" +constant[69].fileref "filedistribution/24d385c5920aaaec/tinybert_encoder_layer_0_output_dense_bias.tbf" +constant[69].type "tensor<float>(d3[312])" +constant[70].name "tinybert_encoder_layer_0_attention_self_value_bias" +constant[70].fileref "filedistribution/9496f2740a93d981/tinybert_encoder_layer_0_attention_self_value_bias.tbf" +constant[70].type "tensor<float>(d2[312])" +constant[71].name "tinybert_Concat_100" +constant[71].fileref "filedistribution/64ea58b13d6a706c/tinybert_Concat_100.tbf" +constant[71].type "tensor<float>(d0[3])" +constant[72].name "tinybert_encoder_layer_3_attention_output_LayerNorm_weight" +constant[72].fileref "filedistribution/d0392f4c85a475f4/tinybert_encoder_layer_3_attention_output_LayerNorm_weight.tbf" +constant[72].type "tensor<float>(d3[312])" +constant[73].name "tinybert_Concat_269" +constant[73].fileref "filedistribution/4f7d38d29831c94c/tinybert_Concat_269.tbf" +constant[73].type "tensor<float>(d0[4])" +constant[74].name "tinybert_encoder_layer_1_attention_output_LayerNorm_weight" +constant[74].fileref "filedistribution/12e62273a701da0c/tinybert_encoder_layer_1_attention_output_LayerNorm_weight.tbf" +constant[74].type "tensor<float>(d3[312])" +constant[75].name "tinybert_encoder_layer_2_attention_output_LayerNorm_weight" +constant[75].fileref "filedistribution/f427490c7481e911/tinybert_encoder_layer_2_attention_output_LayerNorm_weight.tbf" +constant[75].type "tensor<float>(d3[312])" +constant[76].name "tinybert_encoder_layer_3_output_LayerNorm_weight" +constant[76].fileref "filedistribution/8ee0f683d1a850c5/tinybert_encoder_layer_3_output_LayerNorm_weight.tbf" +constant[76].type "tensor<float>(d3[312])" +constant[77].name "tinybert_embeddings_word_embeddings_weight" +constant[77].fileref "filedistribution/f65178b864c98e80/tinybert_embeddings_word_embeddings_weight.tbf" +constant[77].type "tensor<float>(d0[30522],d3[312])" +constant[78].name "tinybert_646" +constant[78].fileref "filedistribution/923e51a03f5d6c88/tinybert_646.tbf" +constant[78].type "tensor<float>(d2[312],d3[312])" +constant[79].name "tinybert_Concat_165" +constant[79].fileref "filedistribution/65461c703dbdae0c/tinybert_Concat_165.tbf" +constant[79].type "tensor<float>(d0[4])" +constant[80].name "tinybert_encoder_layer_3_attention_self_key_bias" +constant[80].fileref "filedistribution/86bda18083e89158/tinybert_encoder_layer_3_attention_self_key_bias.tbf" +constant[80].type "tensor<float>(d2[312])" +constant[81].name "tinybert_Concat_154" +constant[81].fileref "filedistribution/5fa74195044c3fcd/tinybert_Concat_154.tbf" +constant[81].type "tensor<float>(d0[4])" +constant[82].name "tinybert_encoder_layer_0_attention_output_LayerNorm_weight" +constant[82].fileref "filedistribution/e6bf3f48a1e9269e/tinybert_encoder_layer_0_attention_output_LayerNorm_weight.tbf" +constant[82].type "tensor<float>(d3[312])" +constant[83].name "tinybert_encoder_layer_1_attention_self_value_bias" +constant[83].fileref "filedistribution/f69aa2c5d6e9928d/tinybert_encoder_layer_1_attention_self_value_bias.tbf" +constant[83].type "tensor<float>(d2[312])" +constant[84].name "tinybert_657" +constant[84].fileref "filedistribution/9ee8715bbc085dfb/tinybert_657.tbf" +constant[84].type "tensor<float>(d2[1200],d3[312])" +constant[85].name "tinybert_599" +constant[85].fileref "filedistribution/ff781c1ad6f97395/tinybert_599.tbf" +constant[85].type "tensor<float>(d2[312],d3[312])" +constant[86].name "tinybert_654" +constant[86].fileref "filedistribution/f4fd0f62da4d01e4/tinybert_654.tbf" +constant[86].type "tensor<float>(d2[312],d3[312])" diff --git a/model-evaluation/src/test/resources/config/tinybert/ranking-expressions.cfg b/model-evaluation/src/test/resources/config/tinybert/ranking-expressions.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/model-evaluation/src/test/resources/config/tinybert/ranking-expressions.cfg diff --git a/model-integration/pom.xml b/model-integration/pom.xml index 1302984a314..9bb60827a68 100644 --- a/model-integration/pom.xml +++ b/model-integration/pom.xml @@ -69,6 +69,12 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>component</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>net.java.dev.jna</groupId> <artifactId>jna</artifactId> <scope>provided</scope> @@ -105,6 +111,21 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <scope>test</scope> diff --git a/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java b/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java index 002350ce3cf..b40e2b5be72 100644 --- a/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java +++ b/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java @@ -2,8 +2,10 @@ package ai.vespa.embedding; import ai.vespa.modelintegration.evaluator.OnnxEvaluator; import ai.vespa.modelintegration.evaluator.OnnxEvaluatorOptions; -import com.yahoo.embedding.BertBaseEmbedderConfig; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; +import com.yahoo.component.AbstractComponent; import com.yahoo.component.annotation.Inject; +import com.yahoo.embedding.BertBaseEmbedderConfig; import com.yahoo.language.process.Embedder; import com.yahoo.language.wordpiece.WordPieceEmbedder; import com.yahoo.tensor.IndexedTensor; @@ -28,7 +30,7 @@ import java.util.Map; * * @author lesters */ -public class BertBaseEmbedder implements Embedder { +public class BertBaseEmbedder extends AbstractComponent implements Embedder { private final static int TOKEN_CLS = 101; // [CLS] private final static int TOKEN_SEP = 102; // [SEP] @@ -44,7 +46,7 @@ public class BertBaseEmbedder implements Embedder { private final OnnxEvaluator evaluator; @Inject - public BertBaseEmbedder(BertBaseEmbedderConfig config) { + public BertBaseEmbedder(OnnxRuntime onnx, BertBaseEmbedderConfig config) { maxTokens = config.transformerMaxTokens(); inputIdsName = config.transformerInputIds(); attentionMaskName = config.transformerAttentionMask(); @@ -58,7 +60,7 @@ public class BertBaseEmbedder implements Embedder { options.setIntraOpThreads(modifyThreadCount(config.onnxIntraOpThreads())); tokenizer = new WordPieceEmbedder.Builder(config.tokenizerVocab().toString()).build(); - evaluator = new OnnxEvaluator(config.transformerModel().toString(), options); + this.evaluator = onnx.evaluatorOf(config.transformerModel().toString(), options); validateModel(); } @@ -100,6 +102,8 @@ public class BertBaseEmbedder implements Embedder { return embedTokens(tokens, type); } + @Override public void deconstruct() { evaluator.close(); } + Tensor embedTokens(List<Integer> tokens, TensorType type) { Tensor inputSequence = createTensorRepresentation(tokens, "d1"); Tensor attentionMask = createAttentionMask(inputSequence); diff --git a/model-integration/src/main/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedder.java b/model-integration/src/main/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedder.java index 81150fe99b0..9572cfcb0e4 100644 --- a/model-integration/src/main/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedder.java +++ b/model-integration/src/main/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedder.java @@ -3,22 +3,25 @@ package ai.vespa.embedding.huggingface; import ai.djl.huggingface.tokenizers.Encoding; import ai.djl.huggingface.tokenizers.HuggingFaceTokenizer; import ai.vespa.modelintegration.evaluator.OnnxEvaluator; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; +import com.yahoo.component.AbstractComponent; import com.yahoo.component.annotation.Inject; +import com.yahoo.embedding.huggingface.HuggingFaceEmbedderConfig; import com.yahoo.language.process.Embedder; import com.yahoo.tensor.IndexedTensor; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorAddress; import com.yahoo.tensor.TensorType; -import com.yahoo.embedding.huggingface.HuggingFaceEmbedderConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.io.*; +import java.io.IOException; import java.nio.file.Paths; -import java.util.*; +import java.util.Arrays; +import java.util.List; +import java.util.Map; -import org.slf4j.LoggerFactory; -import org.slf4j.Logger; - -public class HuggingFaceEmbedder implements Embedder { +public class HuggingFaceEmbedder extends AbstractComponent implements Embedder { private static final Logger LOG = LoggerFactory.getLogger(HuggingFaceEmbedder.class.getName()); @@ -30,7 +33,7 @@ public class HuggingFaceEmbedder implements Embedder { private final OnnxEvaluator evaluator; @Inject - public HuggingFaceEmbedder(HuggingFaceEmbedderConfig config) throws IOException { + public HuggingFaceEmbedder(OnnxRuntime onnx, HuggingFaceEmbedderConfig config) throws IOException { maxTokens = config.transformerMaxTokens(); inputIdsName = config.transformerInputIds(); attentionMaskName = config.transformerAttentionMask(); @@ -48,7 +51,7 @@ public class HuggingFaceEmbedder implements Embedder { LOG.info("Could not initialize the tokenizer"); throw new IOException("Could not initialize the tokenizer."); } - evaluator = new OnnxEvaluator(config.transformerModel().toString()); + evaluator = onnx.evaluatorOf(config.transformerModel().toString()); validateModel(); } @@ -83,6 +86,8 @@ public class HuggingFaceEmbedder implements Embedder { return tokenIds; } + @Override public void deconstruct() { evaluator.close(); } + public List<Integer> longToInteger(long[] values) { return Arrays.stream(values) .boxed().map(Long::intValue) diff --git a/model-integration/src/main/java/ai/vespa/llm/Generator.java b/model-integration/src/main/java/ai/vespa/llm/Generator.java index ed231a5e94c..973b5ac2899 100644 --- a/model-integration/src/main/java/ai/vespa/llm/Generator.java +++ b/model-integration/src/main/java/ai/vespa/llm/Generator.java @@ -2,6 +2,8 @@ package ai.vespa.llm; import ai.vespa.modelintegration.evaluator.OnnxEvaluator; import ai.vespa.modelintegration.evaluator.OnnxEvaluatorOptions; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; +import com.yahoo.component.AbstractComponent; import com.yahoo.component.annotation.Inject; import com.yahoo.language.process.Embedder; import com.yahoo.language.sentencepiece.SentencePieceEmbedder; @@ -25,7 +27,7 @@ import java.util.Map; * * @author lesters */ -public class Generator { +public class Generator extends AbstractComponent { private final static int TOKEN_EOS = 1; // end of sequence @@ -46,7 +48,7 @@ public class Generator { private final OnnxEvaluator decoder; @Inject - public Generator(GeneratorConfig config) { + public Generator(OnnxRuntime onnx, GeneratorConfig config) { // Set up tokenizer tokenizer = new SentencePieceEmbedder.Builder(config.tokenizerModel().toString()).build(); tokenizerMaxTokens = config.tokenizerMaxTokens(); @@ -61,7 +63,7 @@ public class Generator { encoderOptions.setInterOpThreads(modifyThreadCount(config.encoderOnnxInterOpThreads())); encoderOptions.setIntraOpThreads(modifyThreadCount(config.encoderOnnxIntraOpThreads())); - encoder = new OnnxEvaluator(config.encoderModel().toString(), encoderOptions); + encoder = onnx.evaluatorOf(config.encoderModel().toString(), encoderOptions); // Set up decoder decoderInputIdsName = config.decoderModelInputIdsName(); @@ -74,7 +76,7 @@ public class Generator { decoderOptions.setInterOpThreads(modifyThreadCount(config.decoderOnnxInterOpThreads())); decoderOptions.setIntraOpThreads(modifyThreadCount(config.decoderOnnxIntraOpThreads())); - decoder = new OnnxEvaluator(config.decoderModel().toString(), decoderOptions); + decoder = onnx.evaluatorOf(config.decoderModel().toString(), decoderOptions); validateModels(); } @@ -99,6 +101,8 @@ public class Generator { return generate(prompt, new GeneratorOptions()); } + @Override public void deconstruct() { encoder.close(); decoder.close(); } + private String generateNotImplemented(GeneratorOptions options) { throw new UnsupportedOperationException("Search method '" + options.getSearchMethod() + "' is currently not implemented"); } diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java index 9961c24005c..7cdc27b6d63 100644 --- a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java +++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java @@ -2,11 +2,12 @@ package ai.vespa.modelintegration.evaluator; +import ai.onnxruntime.NodeInfo; import ai.onnxruntime.OnnxTensor; import ai.onnxruntime.OnnxValue; -import ai.onnxruntime.OrtEnvironment; import ai.onnxruntime.OrtException; import ai.onnxruntime.OrtSession; +import ai.vespa.modelintegration.evaluator.OnnxRuntime.ReferencedOrtSession; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; @@ -14,32 +15,28 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import static ai.vespa.modelintegration.evaluator.OnnxRuntime.isCudaError; + /** * Evaluates an ONNX Model by deferring to ONNX Runtime. * * @author lesters */ -public class OnnxEvaluator { - - private final OrtEnvironment environment; - private final OrtSession session; +public class OnnxEvaluator implements AutoCloseable { - public OnnxEvaluator(String modelPath) { - this(modelPath, null); - } + private final ReferencedOrtSession session; - public OnnxEvaluator(String modelPath, OnnxEvaluatorOptions options) { - environment = OrtEnvironment.getEnvironment(); - session = createSession(modelPath, environment, options, true); + OnnxEvaluator(String modelPath, OnnxEvaluatorOptions options, OnnxRuntime runtime) { + session = createSession(modelPath, runtime, options, true); } public Tensor evaluate(Map<String, Tensor> inputs, String output) { Map<String, OnnxTensor> onnxInputs = null; try { output = mapToInternalName(output); - onnxInputs = TensorConverter.toOnnxTensors(inputs, environment, session); - try (OrtSession.Result result = session.run(onnxInputs, Collections.singleton(output))) { + onnxInputs = TensorConverter.toOnnxTensors(inputs, OnnxRuntime.ortEnvironment(), session.instance()); + try (OrtSession.Result result = session.instance().run(onnxInputs, Collections.singleton(output))) { return TensorConverter.toVespaTensor(result.get(0)); } } catch (OrtException e) { @@ -54,9 +51,9 @@ public class OnnxEvaluator { public Map<String, Tensor> evaluate(Map<String, Tensor> inputs) { Map<String, OnnxTensor> onnxInputs = null; try { - onnxInputs = TensorConverter.toOnnxTensors(inputs, environment, session); + onnxInputs = TensorConverter.toOnnxTensors(inputs, OnnxRuntime.ortEnvironment(), session.instance()); Map<String, Tensor> outputs = new HashMap<>(); - try (OrtSession.Result result = session.run(onnxInputs)) { + try (OrtSession.Result result = session.instance().run(onnxInputs)) { for (Map.Entry<String, OnnxValue> output : result) { String mapped = TensorConverter.asValidName(output.getKey()); outputs.put(mapped, TensorConverter.toVespaTensor(output.getValue())); @@ -72,9 +69,38 @@ public class OnnxEvaluator { } } + public record IdAndType(String id, TensorType type) { } + + private Map<String, IdAndType> toSpecMap(Map<String, NodeInfo> infoMap) { + Map<String, IdAndType> result = new HashMap<>(); + for (var info : infoMap.entrySet()) { + String name = info.getKey(); + String ident = TensorConverter.asValidName(name); + TensorType t = TensorConverter.toVespaType(info.getValue().getInfo()); + result.put(name, new IdAndType(ident, t)); + } + return result; + } + + public Map<String, IdAndType> getInputs() { + try { + return toSpecMap(session.instance().getInputInfo()); + } catch (OrtException e) { + throw new RuntimeException("ONNX Runtime exception", e); + } + } + + public Map<String, IdAndType> getOutputs() { + try { + return toSpecMap(session.instance().getOutputInfo()); + } catch (OrtException e) { + throw new RuntimeException("ONNX Runtime exception", e); + } + } + public Map<String, TensorType> getInputInfo() { try { - return TensorConverter.toVespaTypes(session.getInputInfo()); + return TensorConverter.toVespaTypes(session.instance().getInputInfo()); } catch (OrtException e) { throw new RuntimeException("ONNX Runtime exception", e); } @@ -82,25 +108,36 @@ public class OnnxEvaluator { public Map<String, TensorType> getOutputInfo() { try { - return TensorConverter.toVespaTypes(session.getOutputInfo()); + return TensorConverter.toVespaTypes(session.instance().getOutputInfo()); } catch (OrtException e) { throw new RuntimeException("ONNX Runtime exception", e); } } - private static OrtSession createSession(String modelPath, OrtEnvironment environment, OnnxEvaluatorOptions options, boolean tryCuda) { + @Override + public void close() throws IllegalStateException { + try { + session.close(); + } catch (UncheckedOrtException e) { + throw new IllegalStateException("Failed to close ONNX session", e); + } catch (IllegalStateException e) { + throw new IllegalStateException("Already closed", e); + } + } + + private static ReferencedOrtSession createSession(String modelPath, OnnxRuntime runtime, OnnxEvaluatorOptions options, boolean tryCuda) { if (options == null) { options = new OnnxEvaluatorOptions(); } try { - return environment.createSession(modelPath, options.getOptions(tryCuda && options.requestingGpu())); + return runtime.acquireSession(modelPath, options, tryCuda && options.requestingGpu()); } catch (OrtException e) { if (e.getCode() == OrtException.OrtErrorCode.ORT_NO_SUCHFILE) { throw new IllegalArgumentException("No such file: " + modelPath); } if (tryCuda && isCudaError(e) && !options.gpuDeviceRequired()) { // Failed in CUDA native code, but GPU device is optional, so we can proceed without it - return createSession(modelPath, environment, options, false); + return createSession(modelPath, runtime, options, false); } if (isCudaError(e)) { throw new IllegalArgumentException("GPU device is required, but CUDA initialization failed", e); @@ -109,34 +146,8 @@ public class OnnxEvaluator { } } - private static boolean isCudaError(OrtException e) { - return switch (e.getCode()) { - case ORT_FAIL -> e.getMessage().contains("cudaError"); - case ORT_EP_FAIL -> e.getMessage().contains("Failed to find CUDA"); - default -> false; - }; - } - - public static boolean isRuntimeAvailable() { - return isRuntimeAvailable(""); - } - - public static boolean isRuntimeAvailable(String modelPath) { - try { - new OnnxEvaluator(modelPath); - return true; - } catch (IllegalArgumentException e) { - if (e.getMessage().equals("No such file: ")) { - return true; - } - return false; - } catch (UnsatisfiedLinkError | RuntimeException | NoClassDefFoundError e) { - return false; - } - } - private String mapToInternalName(String outputName) throws OrtException { - var info = session.getOutputInfo(); + var info = session.instance().getOutputInfo(); var internalNames = info.keySet(); for (String name : internalNames) { if (name.equals(outputName)) { diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorOptions.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorOptions.java index b6de9698f1a..1ed219a8560 100644 --- a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorOptions.java +++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorOptions.java @@ -5,6 +5,8 @@ package ai.vespa.modelintegration.evaluator; import ai.onnxruntime.OrtException; import ai.onnxruntime.OrtSession; +import java.util.Objects; + /** * Session options for ONNX Runtime evaluation * @@ -12,7 +14,7 @@ import ai.onnxruntime.OrtSession; */ public class OnnxEvaluatorOptions { - private OrtSession.SessionOptions.OptLevel optimizationLevel; + private final OrtSession.SessionOptions.OptLevel optimizationLevel; private OrtSession.SessionOptions.ExecutionMode executionMode; private int interOpThreads; private int intraOpThreads; @@ -74,4 +76,18 @@ public class OnnxEvaluatorOptions { return gpuDeviceRequired; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + OnnxEvaluatorOptions that = (OnnxEvaluatorOptions) o; + return interOpThreads == that.interOpThreads && intraOpThreads == that.intraOpThreads + && gpuDeviceNumber == that.gpuDeviceNumber && gpuDeviceRequired == that.gpuDeviceRequired + && optimizationLevel == that.optimizationLevel && executionMode == that.executionMode; + } + + @Override + public int hashCode() { + return Objects.hash(optimizationLevel, executionMode, interOpThreads, intraOpThreads, gpuDeviceNumber, gpuDeviceRequired); + } } diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java new file mode 100644 index 00000000000..42830041c02 --- /dev/null +++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java @@ -0,0 +1,170 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package ai.vespa.modelintegration.evaluator; + +import ai.onnxruntime.OrtEnvironment; +import ai.onnxruntime.OrtException; +import ai.onnxruntime.OrtSession; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.annotation.Inject; +import com.yahoo.jdisc.ResourceReference; +import com.yahoo.jdisc.refcount.DebugReferencesWithStack; +import com.yahoo.jdisc.refcount.References; + +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static com.yahoo.yolean.Exceptions.throwUnchecked; + +/** + * Provides ONNX runtime environment with session management. + * + * @author bjorncs + */ +public class OnnxRuntime extends AbstractComponent { + + // For unit testing + @FunctionalInterface interface OrtSessionFactory { + OrtSession create(String path, OrtSession.SessionOptions opts) throws OrtException; + } + + private static final Logger log = Logger.getLogger(OnnxRuntime.class.getName()); + + private static final OrtEnvironmentResult ortEnvironment = getOrtEnvironment(); + private static final OrtSessionFactory defaultFactory = (path, opts) -> ortEnvironment().createSession(path, opts); + + private final Object monitor = new Object(); + private final Map<OrtSessionId, SharedOrtSession> sessions = new HashMap<>(); + private final OrtSessionFactory factory; + + @Inject public OnnxRuntime() { this(defaultFactory); } + + OnnxRuntime(OrtSessionFactory factory) { this.factory = factory; } + + public OnnxEvaluator evaluatorOf(String modelPath) { + return new OnnxEvaluator(modelPath, null, this); + } + + public OnnxEvaluator evaluatorOf(String modelPath, OnnxEvaluatorOptions options) { + return new OnnxEvaluator(modelPath, options, this); + } + + public static OrtEnvironment ortEnvironment() { + if (ortEnvironment.env() != null) return ortEnvironment.env(); + throw throwUnchecked(ortEnvironment.failure()); + } + + @Override + public void deconstruct() { + synchronized (monitor) { + sessions.forEach((id, sharedSession) -> { + int hash = System.identityHashCode(sharedSession.session()); + var refs = sharedSession.references(); + log.warning("Closing leaked session %s (%s) with %d outstanding references:\n%s" + .formatted(id, hash, refs.referenceCount(), refs.currentState())); + try { + sharedSession.session().close(); + } catch (Exception e) { + log.log(Level.WARNING, "Failed to close session %s (%s)".formatted(id, hash), e); + } + }); + sessions.clear(); + } + } + + private static OrtEnvironmentResult getOrtEnvironment() { + try { + return new OrtEnvironmentResult(OrtEnvironment.getEnvironment(), null); + } catch (UnsatisfiedLinkError | RuntimeException | NoClassDefFoundError e) { + log.log(Level.FINE, e, () -> "Failed to load ONNX runtime"); + return new OrtEnvironmentResult(null, e); + } + } + + public static boolean isRuntimeAvailable() { return ortEnvironment.env() != null; } + public static boolean isRuntimeAvailable(String modelPath) { + if (!isRuntimeAvailable()) return false; + try { + // Expensive way of checking if runtime is available as it incurs the cost of loading the model if successful + defaultFactory.create(modelPath, new OnnxEvaluatorOptions().getOptions(false)); + return true; + } catch (OrtException e) { + return e.getCode() == OrtException.OrtErrorCode.ORT_NO_SUCHFILE; + } catch (UnsatisfiedLinkError | RuntimeException | NoClassDefFoundError e) { + return false; + } + } + + static boolean isCudaError(OrtException e) { + return switch (e.getCode()) { + case ORT_FAIL -> e.getMessage().contains("cudaError"); + case ORT_EP_FAIL -> e.getMessage().contains("Failed to find CUDA"); + default -> false; + }; + } + + ReferencedOrtSession acquireSession(String modelPath, OnnxEvaluatorOptions options, boolean loadCuda) throws OrtException { + var sessionId = new OrtSessionId(modelPath, options, loadCuda); + synchronized (monitor) { + var sharedSession = sessions.get(sessionId); + if (sharedSession != null) { + return sharedSession.newReference(); + } + } + + // Note: identical models loaded simultaneously will result in duplicate session instances + var session = factory.create(modelPath, options.getOptions(loadCuda)); + log.fine(() -> "Created new session (%s)".formatted(System.identityHashCode(session))); + + var sharedSession = new SharedOrtSession(sessionId, session); + var referencedSession = sharedSession.newReference(); + synchronized (monitor) { sessions.put(sessionId, sharedSession); } + sharedSession.references().release(); // Release initial reference + return referencedSession; + } + + int sessionsCached() { synchronized(monitor) { return sessions.size(); } } + + public static class ReferencedOrtSession implements AutoCloseable { + private final OrtSession instance; + private final ResourceReference ref; + + public ReferencedOrtSession(OrtSession instance, ResourceReference ref) { + this.instance = instance; + this.ref = ref; + } + + public OrtSession instance() { return instance; } + @Override public void close() { ref.close(); } + } + + // Assumes options are never modified after being stored in `onnxSessions` + record OrtSessionId(String modelPath, OnnxEvaluatorOptions options, boolean loadCuda) {} + + record OrtEnvironmentResult(OrtEnvironment env, Throwable failure) {} + + private class SharedOrtSession { + private final OrtSessionId id; + private final OrtSession session; + private final References refs = new DebugReferencesWithStack(this::close); + + SharedOrtSession(OrtSessionId id, OrtSession session) { + this.id = id; + this.session = session; + } + + ReferencedOrtSession newReference() { return new ReferencedOrtSession(session, refs.refer(id)); } + References references() { return refs; } + OrtSession session() { return session; } + + void close() { + try { + synchronized (OnnxRuntime.this.monitor) { sessions.remove(id); } + log.fine(() -> "Closing session (%s)".formatted(System.identityHashCode(session))); + session.close(); + } catch (OrtException e) { throw new UncheckedOrtException(e);} + } + } +} diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/UncheckedOrtException.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/UncheckedOrtException.java new file mode 100644 index 00000000000..1f2c8ba2cf7 --- /dev/null +++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/UncheckedOrtException.java @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package ai.vespa.modelintegration.evaluator; + +import ai.onnxruntime.OrtException; + +/** + * @author bjorncs + */ +public class UncheckedOrtException extends RuntimeException { + + public UncheckedOrtException(Throwable e) { super(e.getMessage(), e); } + + @Override public synchronized OrtException getCause() { return (OrtException) super.getCause(); } +} diff --git a/model-integration/src/test/java/ai/vespa/embedding/BertBaseEmbedderTest.java b/model-integration/src/test/java/ai/vespa/embedding/BertBaseEmbedderTest.java index b06a54d68bb..329b87cacd1 100644 --- a/model-integration/src/test/java/ai/vespa/embedding/BertBaseEmbedderTest.java +++ b/model-integration/src/test/java/ai/vespa/embedding/BertBaseEmbedderTest.java @@ -1,13 +1,12 @@ package ai.vespa.embedding; -import ai.vespa.modelintegration.evaluator.OnnxEvaluator; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import com.yahoo.config.ModelReference; import com.yahoo.embedding.BertBaseEmbedderConfig; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import org.junit.Test; -import java.lang.IllegalArgumentException; import java.util.List; import static org.junit.Assert.assertEquals; @@ -20,12 +19,12 @@ public class BertBaseEmbedderTest { public void testEmbedder() { String vocabPath = "src/test/models/onnx/transformer/dummy_vocab.txt"; String modelPath = "src/test/models/onnx/transformer/dummy_transformer.onnx"; - assumeTrue(OnnxEvaluator.isRuntimeAvailable(modelPath)); + assumeTrue(OnnxRuntime.isRuntimeAvailable(modelPath)); BertBaseEmbedderConfig.Builder builder = new BertBaseEmbedderConfig.Builder(); builder.tokenizerVocab(ModelReference.valueOf(vocabPath)); builder.transformerModel(ModelReference.valueOf(modelPath)); - BertBaseEmbedder embedder = new BertBaseEmbedder(builder.build()); + BertBaseEmbedder embedder = newBertBaseEmbedder(builder.build()); TensorType destType = TensorType.fromSpec("tensor<float>(x[7])"); List<Integer> tokens = List.of(1,2,3,4,5); // use random tokens instead of invoking the tokenizer @@ -39,13 +38,13 @@ public class BertBaseEmbedderTest { public void testEmbedderWithoutTokenTypeIdsName() { String vocabPath = "src/test/models/onnx/transformer/dummy_vocab.txt"; String modelPath = "src/test/models/onnx/transformer/dummy_transformer_without_type_ids.onnx"; - assumeTrue(OnnxEvaluator.isRuntimeAvailable(modelPath)); + assumeTrue(OnnxRuntime.isRuntimeAvailable(modelPath)); BertBaseEmbedderConfig.Builder builder = new BertBaseEmbedderConfig.Builder(); builder.tokenizerVocab(ModelReference.valueOf(vocabPath)); builder.transformerModel(ModelReference.valueOf(modelPath)); builder.transformerTokenTypeIds(""); - BertBaseEmbedder embedder = new BertBaseEmbedder(builder.build()); + BertBaseEmbedder embedder = newBertBaseEmbedder(builder.build()); TensorType destType = TensorType.fromSpec("tensor<float>(x[7])"); List<Integer> tokens = List.of(1,2,3,4,5); // use random tokens instead of invoking the tokenizer @@ -59,14 +58,18 @@ public class BertBaseEmbedderTest { public void testEmbedderWithoutTokenTypeIdsNameButWithConfig() { String vocabPath = "src/test/models/onnx/transformer/dummy_vocab.txt"; String modelPath = "src/test/models/onnx/transformer/dummy_transformer_without_type_ids.onnx"; - assumeTrue(OnnxEvaluator.isRuntimeAvailable(modelPath)); + assumeTrue(OnnxRuntime.isRuntimeAvailable(modelPath)); BertBaseEmbedderConfig.Builder builder = new BertBaseEmbedderConfig.Builder(); builder.tokenizerVocab(ModelReference.valueOf(vocabPath)); builder.transformerModel(ModelReference.valueOf(modelPath)); // we did not configured BertBaseEmbedder to accept missing token type ids // so we expect ctor to throw - assertThrows(IllegalArgumentException.class, () -> { new BertBaseEmbedder(builder.build()); }); + assertThrows(IllegalArgumentException.class, () -> { newBertBaseEmbedder(builder.build()); }); + } + + private static BertBaseEmbedder newBertBaseEmbedder(BertBaseEmbedderConfig cfg) { + return new BertBaseEmbedder(new OnnxRuntime(), cfg); } } diff --git a/model-integration/src/test/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedderTest.java b/model-integration/src/test/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedderTest.java index c67b6b0dcab..0ff9acc9a69 100644 --- a/model-integration/src/test/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedderTest.java +++ b/model-integration/src/test/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedderTest.java @@ -1,19 +1,5 @@ package ai.vespa.embedding.huggingface; -import ai.vespa.modelintegration.evaluator.OnnxEvaluator; -import com.yahoo.config.ModelReference; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; -import org.junit.Test; - -import com.yahoo.embedding.huggingface.HuggingFaceEmbedderConfig; - -import java.io.IOException; -import java.util.List; - -import static org.junit.Assume.assumeTrue; -import static org.junit.Assert.assertEquals; - public class HuggingFaceEmbedderTest { /* @Test @@ -21,7 +7,7 @@ public class HuggingFaceEmbedderTest { String modelPath = "src/test/models/hf/model.onnx"; String tokenizerPath = "src/test/models/hf/tokenizer.json"; - assumeTrue(OnnxEvaluator.isRuntimeAvailable(modelPath)); + assumeTrue(OnnxRuntime.isRuntimeAvailable(modelPath)); HuggingFaceEmbedderConfig.Builder builder = new HuggingFaceEmbedderConfig.Builder(); builder.tokenizerPath(ModelReference.valueOf(tokenizerPath)); diff --git a/model-integration/src/test/java/ai/vespa/llm/GeneratorTest.java b/model-integration/src/test/java/ai/vespa/llm/GeneratorTest.java index 733430aa10d..c22902b344f 100644 --- a/model-integration/src/test/java/ai/vespa/llm/GeneratorTest.java +++ b/model-integration/src/test/java/ai/vespa/llm/GeneratorTest.java @@ -1,6 +1,6 @@ package ai.vespa.llm; -import ai.vespa.modelintegration.evaluator.OnnxEvaluator; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import com.yahoo.config.ModelReference; import com.yahoo.llm.GeneratorConfig; import org.junit.Test; @@ -15,13 +15,13 @@ public class GeneratorTest { String vocabPath = "src/test/models/onnx/llm/en.wiki.bpe.vs10000.model"; String encoderModelPath = "src/test/models/onnx/llm/random_encoder.onnx"; String decoderModelPath = "src/test/models/onnx/llm/random_decoder.onnx"; - assumeTrue(OnnxEvaluator.isRuntimeAvailable(encoderModelPath)); + assumeTrue(OnnxRuntime.isRuntimeAvailable(encoderModelPath)); GeneratorConfig.Builder builder = new GeneratorConfig.Builder(); builder.tokenizerModel(ModelReference.valueOf(vocabPath)); builder.encoderModel(ModelReference.valueOf(encoderModelPath)); builder.decoderModel(ModelReference.valueOf(decoderModelPath)); - Generator generator = new Generator(builder.build()); + Generator generator = newGenerator(builder.build()); GeneratorOptions options = new GeneratorOptions(); options.setSearchMethod(GeneratorOptions.SearchMethod.GREEDY); @@ -33,4 +33,8 @@ public class GeneratorTest { assertEquals("<unk> linear recruit latest sack annually institutions cert solid references", result); } + private static Generator newGenerator(GeneratorConfig cfg) { + return new Generator(new OnnxRuntime(), cfg); + } + } diff --git a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java index 83f355821e5..5aba54de11b 100644 --- a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java +++ b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java @@ -5,23 +5,31 @@ package ai.vespa.modelintegration.evaluator; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import org.junit.Test; +import org.junit.jupiter.api.BeforeAll; import java.util.HashMap; import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeTrue; +import static org.junit.Assume.assumeNotNull; /** * @author lesters */ public class OnnxEvaluatorTest { + private static OnnxRuntime runtime; + + @BeforeAll + public static void beforeAll() { + if (OnnxRuntime.isRuntimeAvailable()) runtime = new OnnxRuntime(); + } + @Test public void testSimpleModel() { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); - OnnxEvaluator evaluator = new OnnxEvaluator("src/test/models/onnx/simple/simple.onnx"); + assumeNotNull(runtime); + OnnxEvaluator evaluator = runtime.evaluatorOf("src/test/models/onnx/simple/simple.onnx"); // Input types Map<String, TensorType> inputTypes = evaluator.getInputInfo(); @@ -45,8 +53,8 @@ public class OnnxEvaluatorTest { @Test public void testBatchDimension() { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); - OnnxEvaluator evaluator = new OnnxEvaluator("src/test/models/onnx/pytorch/one_layer.onnx"); + assumeNotNull(runtime); + OnnxEvaluator evaluator = runtime.evaluatorOf("src/test/models/onnx/pytorch/one_layer.onnx"); // Input types Map<String, TensorType> inputTypes = evaluator.getInputInfo(); @@ -64,7 +72,7 @@ public class OnnxEvaluatorTest { @Test public void testMatMul() { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + assumeNotNull(runtime); String expected = "tensor<float>(d0[2],d1[4]):[38,44,50,56,83,98,113,128]"; String input1 = "tensor<float>(d0[2],d1[3]):[1,2,3,4,5,6]"; String input2 = "tensor<float>(d0[3],d1[4]):[1,2,3,4,5,6,7,8,9,10,11,12]"; @@ -73,7 +81,7 @@ public class OnnxEvaluatorTest { @Test public void testTypes() { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + assumeNotNull(runtime); assertEvaluate("add_double.onnx", "tensor(d0[1]):[3]", "tensor(d0[1]):[1]", "tensor(d0[1]):[2]"); assertEvaluate("add_float.onnx", "tensor<float>(d0[1]):[3]", "tensor<float>(d0[1]):[1]", "tensor<float>(d0[1]):[2]"); assertEvaluate("add_int64.onnx", "tensor<double>(d0[1]):[3]", "tensor<double>(d0[1]):[1]", "tensor<double>(d0[1]):[2]"); @@ -86,8 +94,8 @@ public class OnnxEvaluatorTest { @Test public void testNotIdentifiers() { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); - OnnxEvaluator evaluator = new OnnxEvaluator("src/test/models/onnx/badnames.onnx"); + assumeNotNull(runtime); + OnnxEvaluator evaluator = runtime.evaluatorOf("src/test/models/onnx/badnames.onnx"); var inputInfo = evaluator.getInputInfo(); var outputInfo = evaluator.getOutputInfo(); for (var entry : inputInfo.entrySet()) { @@ -152,7 +160,7 @@ public class OnnxEvaluatorTest { } private void assertEvaluate(String model, String output, String... input) { - OnnxEvaluator evaluator = new OnnxEvaluator("src/test/models/onnx/" + model); + OnnxEvaluator evaluator = runtime.evaluatorOf("src/test/models/onnx/" + model); Map<String, Tensor> inputs = new HashMap<>(); for (int i = 0; i < input.length; ++i) { inputs.put("input" + (i+1), Tensor.from(input[i])); diff --git a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxRuntimeTest.java b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxRuntimeTest.java new file mode 100644 index 00000000000..81b1237e770 --- /dev/null +++ b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxRuntimeTest.java @@ -0,0 +1,48 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package ai.vespa.modelintegration.evaluator; + +import ai.onnxruntime.OrtException; +import ai.onnxruntime.OrtSession; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +/** + * @author bjorncs + */ +class OnnxRuntimeTest { + + @Test + void reuses_sessions_while_active() throws OrtException { + var runtime = new OnnxRuntime((__, ___) -> mock(OrtSession.class)); + var session1 = runtime.acquireSession("model1", new OnnxEvaluatorOptions(), false); + var session2 = runtime.acquireSession("model1", new OnnxEvaluatorOptions(), false); + var session3 = runtime.acquireSession("model2", new OnnxEvaluatorOptions(), false); + assertSame(session1.instance(), session2.instance()); + assertNotSame(session1.instance(), session3.instance()); + assertEquals(2, runtime.sessionsCached()); + + session1.close(); + session2.close(); + assertEquals(1, runtime.sessionsCached()); + verify(session1.instance()).close(); + verify(session3.instance(), never()).close(); + + session3.close(); + assertEquals(0, runtime.sessionsCached()); + verify(session3.instance()).close(); + + var session4 = runtime.acquireSession("model1", new OnnxEvaluatorOptions(), false); + assertNotSame(session1.instance(), session4.instance()); + assertEquals(1, runtime.sessionsCached()); + session4.close(); + assertEquals(0, runtime.sessionsCached()); + verify(session4.instance()).close(); + } +}
\ No newline at end of file diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java index 61ee612e3de..6e6a6dec9d3 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java @@ -249,6 +249,7 @@ public class ConfigServerApiImpl implements ConfigServerApi { return HttpClientBuilder.create() .setDefaultRequestConfig(DEFAULT_REQUEST_CONFIG) .disableAutomaticRetries() + .disableConnectionState() // Share connections between subsequent requests. .setUserAgent("node-admin") .setConnectionManager(cm) .build(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index fc49dcc744c..15be7accb7d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -41,6 +41,7 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; @@ -63,11 +64,10 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private final URI ztsEndpoint; private final Path ztsTrustStorePath; - private final AthenzIdentity configserverIdentity; private final Clock clock; + private final String certificateDnsSuffix; private final ServiceIdentityProvider hostIdentityProvider; private final IdentityDocumentClient identityDocumentClient; - private final CsrGenerator csrGenerator; private final boolean useInternalZts; // Used as an optimization to ensure ZTS is not DDoS'ed on continuously failing refresh attempts @@ -82,13 +82,12 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { Clock clock) { this.ztsEndpoint = ztsEndpoint; this.ztsTrustStorePath = ztsTrustStorePath; - this.configserverIdentity = configServerInfo.getConfigServerIdentity(); - this.csrGenerator = new CsrGenerator(certificateDnsSuffix, configserverIdentity.getFullName()); + this.certificateDnsSuffix = certificateDnsSuffix; this.hostIdentityProvider = hostIdentityProvider; this.identityDocumentClient = new DefaultIdentityDocumentClient( configServerInfo.getLoadBalancerEndpoint(), hostIdentityProvider, - new AthenzIdentityVerifier(Set.of(configserverIdentity))); + new AthenzIdentityVerifier(Set.of(configServerInfo.getConfigServerIdentity()))); this.clock = clock; this.useInternalZts = useInternalZts; } @@ -185,18 +184,17 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private void registerIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile, ContainerPath identityDocumentFile) { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); - var doc = identityDocumentClient.getNodeIdentityDocument(context.hostname().value()); + SignedIdentityDocument doc = identityDocumentClient.getNodeIdentityDocument(context.hostname().value()); + CsrGenerator csrGenerator = new CsrGenerator(certificateDnsSuffix, doc.providerService().getFullName()); Pkcs10Csr csr = csrGenerator.generateInstanceCsr( context.identity(), doc.providerUniqueId(), doc.ipAddresses(), doc.clusterType(), keyPair); - // Set up a hostname verified for zts if this is configured to use the config server (internal zts) apis - HostnameVerifier ztsHostNameVerifier = useInternalZts - ? new AthenzIdentityVerifier(Set.of(configserverIdentity)) - : null; - try (ZtsClient ztsClient = new DefaultZtsClient.Builder(ztsEndpoint).withIdentityProvider(hostIdentityProvider).withHostnameVerifier(ztsHostNameVerifier).build()) { + // Allow all zts hosts while removing SIS + HostnameVerifier ztsHostNameVerifier = (hostname, sslSession) -> true; + try (ZtsClient ztsClient = new DefaultZtsClient.Builder(ztsEndpoint(doc)).withIdentityProvider(hostIdentityProvider).withHostnameVerifier(ztsHostNameVerifier).build()) { InstanceIdentity instanceIdentity = ztsClient.registerInstance( - configserverIdentity, + doc.providerService(), context.identity(), EntityBindingsMapper.toAttestationData(doc), csr); @@ -206,9 +204,19 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { } } + /** + * Return zts url from identity document, fallback to ztsEndpoint + */ + private URI ztsEndpoint(SignedIdentityDocument doc) { + return Optional.ofNullable(doc.ztsUrl()) + .filter(s -> !s.isBlank()) + .map(URI::create) + .orElse(ztsEndpoint); + } private void refreshIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile, ContainerPath identityDocumentFile, SignedIdentityDocument doc) { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); + CsrGenerator csrGenerator = new CsrGenerator(certificateDnsSuffix, doc.providerService().getFullName()); Pkcs10Csr csr = csrGenerator.generateInstanceCsr( context.identity(), doc.providerUniqueId(), doc.ipAddresses(), doc.clusterType(), keyPair); @@ -217,14 +225,12 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { .build(); try { - // Set up a hostname verified for zts if this is configured to use the config server (internal zts) apis - HostnameVerifier ztsHostNameVerifier = useInternalZts - ? new AthenzIdentityVerifier(Set.of(configserverIdentity)) - : null; - try (ZtsClient ztsClient = new DefaultZtsClient.Builder(ztsEndpoint).withSslContext(containerIdentitySslContext).withHostnameVerifier(ztsHostNameVerifier).build()) { + // Allow all zts hosts while removing SIS + HostnameVerifier ztsHostNameVerifier = (hostname, sslSession) -> true; + try (ZtsClient ztsClient = new DefaultZtsClient.Builder(ztsEndpoint(doc)).withSslContext(containerIdentitySslContext).withHostnameVerifier(ztsHostNameVerifier).build()) { InstanceIdentity instanceIdentity = ztsClient.refreshInstance( - configserverIdentity, + doc.providerService(), context.identity(), doc.providerUniqueId().asDottedString(), csr); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java index d884c608ead..e2a7f9167ac 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java @@ -61,4 +61,6 @@ public interface NodeAgentContext extends TaskContext { double vcpuOnThisHost(); Optional<ApplicationId> hostExclusiveTo(); + + boolean exclave(); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java index ed2de691eb0..210bdf2fcb3 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java @@ -45,11 +45,12 @@ public class NodeAgentContextImpl implements NodeAgentContext { private final double cpuSpeedup; private final Set<NodeAgentTask> disabledNodeAgentTasks; private final Optional<ApplicationId> hostExclusiveTo; + private final boolean exclave; public NodeAgentContextImpl(NodeSpec node, Acl acl, AthenzIdentity identity, ContainerNetworkMode containerNetworkMode, ZoneApi zone, FlagSource flagSource, UserScope userScope, PathScope pathScope, - double cpuSpeedup, Optional<ApplicationId> hostExclusiveTo) { + double cpuSpeedup, Optional<ApplicationId> hostExclusiveTo, boolean exclave) { if (cpuSpeedup <= 0) throw new IllegalArgumentException("cpuSpeedUp must be positive, was: " + cpuSpeedup); @@ -68,6 +69,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { .with(FetchVector.Dimension.HOSTNAME, node.hostname()) .with(FetchVector.Dimension.NODE_TYPE, node.type().name()).value()); this.hostExclusiveTo = hostExclusiveTo; + this.exclave = exclave; } @Override @@ -140,6 +142,11 @@ public class NodeAgentContextImpl implements NodeAgentContext { logger.log(level, logPrefix + message, throwable); } + @Override + public boolean exclave() { + return exclave; + } + public static NodeAgentContextImpl.Builder builder(NodeSpec node) { return new Builder(new NodeSpec.Builder(node)); } @@ -168,6 +175,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { private FlagSource flagSource; private double cpuSpeedUp = 1; private Optional<ApplicationId> hostExclusiveTo = Optional.empty(); + private boolean exclave = false; private Builder(NodeSpec.Builder nodeSpecBuilder) { this.nodeSpecBuilder = nodeSpecBuilder; @@ -234,6 +242,11 @@ public class NodeAgentContextImpl implements NodeAgentContext { return this; } + public Builder exclave(boolean exclave) { + this.exclave = exclave; + return this; + } + public NodeAgentContextImpl build() { Objects.requireNonNull(containerStorage, "Must set one of containerStorage or fileSystem"); @@ -273,7 +286,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { Optional.ofNullable(flagSource).orElseGet(InMemoryFlagSource::new), userScope, new PathScope(containerFs, "/opt/vespa"), - cpuSpeedUp, hostExclusiveTo); + cpuSpeedUp, hostExclusiveTo, exclave); } } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index 8d19925a886..00a805790f1 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.WireguardKey; import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; @@ -67,7 +68,7 @@ public class RealNodeRepositoryTest { for (int i = 0; i < 3; i++) { try { int port = findRandomOpenPort(); - container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(port, CloudAccount.empty), Networking.enable); + container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(port, SystemName.main, CloudAccount.empty), Networking.enable); ConfigServerApi configServerApi = ConfigServerApiImpl.createForTesting( List.of(URI.create("http://127.0.0.1:" + port))); waitForJdiscContainerToServe(configServerApi); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 31283998702..ba99219638b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -116,11 +116,11 @@ public final class Node implements Nodelike { if (state == State.ready && type.isHost()) { requireNonEmpty(ipConfig.primary(), "A " + type + " must have at least one primary IP address in state " + state); - requireNonEmpty(ipConfig.pool().asSet(), "A " + type + " must have a non-empty IP address pool in state " + state); + requireNonEmpty(ipConfig.pool().ipSet(), "A " + type + " must have a non-empty IP address pool in state " + state); } if (parentHostname.isPresent()) { - if (!ipConfig.pool().asSet().isEmpty()) throw new IllegalArgumentException("A child node cannot have an IP address pool"); + if (!ipConfig.pool().ipSet().isEmpty()) throw new IllegalArgumentException("A child node cannot have an IP address pool"); if (modelName.isPresent()) throw new IllegalArgumentException("A child node cannot have model name set"); if (switchHostname.isPresent()) throw new IllegalArgumentException("A child node cannot have switch hostname set"); if (status.wantToRebuild()) throw new IllegalArgumentException("A child node cannot be rebuilt"); @@ -388,7 +388,7 @@ public final class Node implements Nodelike { return with(history.with(new History.Event(History.Event.Type.up, agent, instant))); } - /** Returns whether this node is down, according to its recorded 'down' and 'up' events */ + /** Returns true if we have positive evidence that this node is down. */ public boolean isDown() { Optional<Instant> downAt = history().event(History.Event.Type.down).map(History.Event::at); if (downAt.isEmpty()) return false; @@ -396,6 +396,14 @@ public final class Node implements Nodelike { return !history().hasEventAfter(History.Event.Type.up, downAt.get()); } + /** Returns true if we have positive evidence that this node is up. */ + public boolean isUp() { + Optional<Instant> upAt = history().event(History.Event.Type.up).map(History.Event::at); + if (upAt.isEmpty()) return false; + + return !history().hasEventAfter(History.Event.Type.down, upAt.get()); + } + /** Returns a copy of this with allocation set as specified. <code>node.state</code> is *not* changed. */ public Node allocate(ApplicationId owner, ClusterMembership membership, NodeResources requestedResources, Instant at) { return this.with(new Allocation(owner, membership, requestedResources, new Generation(0, 0), false)) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java index 1cf7bcfa4f2..800cf2150e9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.node.ClusterId; +import com.yahoo.vespa.hosted.provision.node.IP; import java.util.ArrayList; import java.util.Comparator; @@ -329,22 +330,22 @@ public class NodeList extends AbstractFilteringList<Node, NodeList> { /** * Returns the number of unused IP addresses in the pool, assuming any and all unaccounted for hostnames - * in the pool are resolved to exactly 1 IP address (or 2 if dual-stack). + * in the pool are resolved to exactly 1 IP address (or 2 with {@link IP.IpAddresses.Protocol#dualStack}). */ public int eventuallyUnusedIpAddressCount(Node host) { // The count in this method relies on the size of the IP address pool if that's non-empty, // otherwise fall back to the address/hostname pool. - if (host.ipConfig().pool().asSet().isEmpty()) { + if (host.ipConfig().pool().ipSet().isEmpty()) { Set<String> allHostnames = cache().keySet(); - return (int) host.ipConfig().pool().hostnames().stream() - .filter(hostname -> !allHostnames.contains(hostname.value())) + return (int) host.ipConfig().pool().getAddressList().stream() + .filter(address -> !allHostnames.contains(address.hostname())) .count(); } Set<String> allIps = ipCache.updateAndGet(old -> old != null ? old : stream().flatMap(node -> node.ipConfig().primary().stream()) .collect(Collectors.toUnmodifiableSet()) ); - return (int) host.ipConfig().pool().asSet().stream() + return (int) host.ipConfig().pool().ipSet().stream() .filter(address -> !allIps.contains(address)) .count(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index fd75644d914..510c4041efb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -28,7 +28,7 @@ import com.yahoo.vespa.hosted.provision.persistence.CuratorDb; import com.yahoo.vespa.hosted.provision.persistence.DnsNameResolver; import com.yahoo.vespa.hosted.provision.persistence.JobControlFlags; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; -import com.yahoo.vespa.hosted.provision.provisioning.ArchiveUris; +import com.yahoo.vespa.hosted.provision.archive.ArchiveUriManager; import com.yahoo.vespa.hosted.provision.provisioning.ContainerImages; import com.yahoo.vespa.hosted.provision.provisioning.FirmwareChecks; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; @@ -57,7 +57,7 @@ public class NodeRepository extends AbstractComponent { private final InfrastructureVersions infrastructureVersions; private final FirmwareChecks firmwareChecks; private final ContainerImages containerImages; - private final ArchiveUris archiveUris; + private final ArchiveUriManager archiveUriManager; private final JobControl jobControl; private final Applications applications; private final LoadBalancers loadBalancers; @@ -134,7 +134,7 @@ public class NodeRepository extends AbstractComponent { this.infrastructureVersions = new InfrastructureVersions(db); this.firmwareChecks = new FirmwareChecks(db, clock); this.containerImages = new ContainerImages(containerImage, tenantContainerImage, tenantGpuContainerImage); - this.archiveUris = new ArchiveUris(db, zone); + this.archiveUriManager = new ArchiveUriManager(db, zone); this.jobControl = new JobControl(new JobControlFlags(db, flagSource)); this.loadBalancers = new LoadBalancers(db); this.metricsDb = metricsDb; @@ -166,7 +166,7 @@ public class NodeRepository extends AbstractComponent { public ContainerImages containerImages() { return containerImages; } /** Returns the archive URIs to use for nodes in this. */ - public ArchiveUris archiveUris() { return archiveUris; } + public ArchiveUriManager archiveUriManager() { return archiveUriManager; } /** Returns the status of maintenance jobs managed by this. */ public JobControl jobControl() { return jobControl; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index ea35f1e85ff..16016815b7c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.applications; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; @@ -33,6 +34,7 @@ public class Cluster { private final boolean required; private final Autoscaling suggested; private final Autoscaling target; + private final ClusterInfo clusterInfo; private final BcpGroupInfo bcpGroupInfo; /** The maxScalingEvents last scaling events of this, sorted by increasing time (newest last) */ @@ -46,6 +48,7 @@ public class Cluster { boolean required, Autoscaling suggested, Autoscaling target, + ClusterInfo clusterInfo, BcpGroupInfo bcpGroupInfo, List<ScalingEvent> scalingEvents) { this.id = Objects.requireNonNull(id); @@ -60,6 +63,7 @@ public class Cluster { this.target = target.withResources(Optional.empty()); // Delete illegal target else this.target = target; + this.clusterInfo = clusterInfo; this.bcpGroupInfo = Objects.requireNonNull(bcpGroupInfo); this.scalingEvents = List.copyOf(scalingEvents); } @@ -105,6 +109,8 @@ public class Cluster { return true; } + public ClusterInfo clusterInfo() { return clusterInfo; } + /** Returns info about the BCP group of clusters this belongs to. */ public BcpGroupInfo bcpGroupInfo() { return bcpGroupInfo; } @@ -119,19 +125,19 @@ public class Cluster { public Cluster withConfiguration(boolean exclusive, Capacity capacity) { return new Cluster(id, exclusive, capacity.minResources(), capacity.maxResources(), capacity.groupSize(), capacity.isRequired(), - suggested, target, bcpGroupInfo, scalingEvents); + suggested, target, capacity.clusterInfo(), bcpGroupInfo, scalingEvents); } public Cluster withSuggested(Autoscaling suggested) { - return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents); } public Cluster withTarget(Autoscaling target) { - return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents); } public Cluster with(BcpGroupInfo bcpGroupInfo) { - return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents); } /** Add or update (based on "at" time) a scaling event */ @@ -145,7 +151,7 @@ public class Cluster { scalingEvents.add(scalingEvent); prune(scalingEvents); - return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents); } @Override @@ -177,7 +183,7 @@ public class Cluster { public static Cluster create(ClusterSpec.Id id, boolean exclusive, Capacity requested) { return new Cluster(id, exclusive, requested.minResources(), requested.maxResources(), requested.groupSize(), requested.isRequired(), - Autoscaling.empty(), Autoscaling.empty(), BcpGroupInfo.empty(), List.of()); + Autoscaling.empty(), Autoscaling.empty(), requested.clusterInfo(), BcpGroupInfo.empty(), List.of()); } /** The predicted time it will take to rescale this cluster. */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java new file mode 100644 index 00000000000..ed4cab5137c --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java @@ -0,0 +1,87 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.archive; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.lang.CachedSupplier; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.persistence.CuratorDb; + +import java.time.Duration; +import java.util.Optional; +import java.util.function.Function; + +/** + * Thread safe class to get and set archive URI for given account and tenants. + * + * @author freva + */ +public class ArchiveUriManager { + + private static final Duration cacheTtl = Duration.ofMinutes(1); + + private final CuratorDb db; + private final Zone zone; + private final CachedSupplier<ArchiveUris> archiveUris; + + public ArchiveUriManager(CuratorDb db, Zone zone) { + this.db = db; + this.zone = zone; + this.archiveUris = new CachedSupplier<>(db::readArchiveUris, cacheTtl); + } + + public ArchiveUris archiveUris() { + return archiveUris.get(); + } + + /** Returns the archive URI to use for given node */ + public Optional<String> archiveUriFor(Node node) { + if (node.allocation().isEmpty()) return Optional.empty(); + ApplicationId app = node.allocation().get().owner(); + + return Optional.ofNullable(node.cloudAccount().isEnclave(zone) ? + archiveUris.get().accountArchiveUris().get(node.cloudAccount()) : + archiveUris.get().tenantArchiveUris().get(app.tenant())) + .map(uri -> { + StringBuilder sb = new StringBuilder(100).append(uri) + .append(app.tenant().value()).append('/') + .append(app.application().value()).append('/') + .append(app.instance().value()).append('/') + .append(node.allocation().get().membership().cluster().id().value()).append('/'); + + for (char c: node.hostname().toCharArray()) { + if (c == '.') break; + sb.append(c); + } + + return sb.append('/').toString(); + }); + } + + /** Set (or remove, if archiveURI is empty) archive URI to use for given tenant */ + public void setArchiveUri(TenantName tenant, Optional<String> archiveUri) { + setArchiveUri(au -> au.with(tenant, archiveUri)); + } + + /** Set (or remove, if archiveURI is empty) archive URI to use for given account */ + public void setArchiveUri(CloudAccount account, Optional<String> archiveUri) { + if (!account.isEnclave(zone) || account.isUnspecified()) + throw new IllegalArgumentException("Cannot set archive URI for non-enclave account: " + account); + setArchiveUri(au -> au.with(account, archiveUri)); + } + + private void setArchiveUri(Function<ArchiveUris, ArchiveUris> mapper) { + try (Lock lock = db.lockArchiveUris()) { + ArchiveUris archiveUris = db.readArchiveUris(); + ArchiveUris updated = mapper.apply(archiveUris); + if (archiveUris.equals(updated)) return; // No change + + db.writeArchiveUris(updated); + this.archiveUris.invalidate(); // Throw away current cache + } + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUris.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUris.java new file mode 100644 index 00000000000..7ec9bd36744 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUris.java @@ -0,0 +1,45 @@ +package com.yahoo.vespa.hosted.provision.archive; + +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.TenantName; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Pattern; + +public record ArchiveUris(Map<TenantName, String> tenantArchiveUris, Map<CloudAccount, String> accountArchiveUris) { + private static final Pattern validUriPattern = Pattern.compile("[a-z0-9]+://(?:(?:[a-z0-9]+(?:[-_][a-z0-9.]+)*)+/)+"); + + public ArchiveUris(Map<TenantName, String> tenantArchiveUris, Map<CloudAccount, String> accountArchiveUris) { + this.tenantArchiveUris = Map.copyOf(tenantArchiveUris); + this.accountArchiveUris = Map.copyOf(accountArchiveUris); + } + + public ArchiveUris with(TenantName tenant, Optional<String> archiveUri) { + Map<TenantName, String> updated = updateIfChanged(tenantArchiveUris, tenant, archiveUri); + if (updated == tenantArchiveUris) return this; + return new ArchiveUris(updated, accountArchiveUris); + } + + public ArchiveUris with(CloudAccount account, Optional<String> archiveUri) { + Map<CloudAccount, String> updated = updateIfChanged(accountArchiveUris, account, archiveUri); + if (updated == accountArchiveUris) return this; + return new ArchiveUris(tenantArchiveUris, updated); + } + + private static <T> Map<T, String> updateIfChanged(Map<T, String> current, T key, Optional<String> archiveUri) { + archiveUri = archiveUri.map(ArchiveUris::normalizeUri); + if (Optional.ofNullable(current.get(key)).equals(archiveUri)) return current; + Map<T, String> updated = new HashMap<>(current); + archiveUri.ifPresentOrElse(uri -> updated.put(key, uri), () -> updated.remove(key)); + return updated; + } + + static String normalizeUri(String uri) { + if (!uri.endsWith("/")) uri = uri + "/"; + if (!validUriPattern.matcher(uri).matches()) + throw new IllegalArgumentException("Invalid archive URI: " + uri); + return uri; + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java index 3825309e97b..2cc43a1eb33 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java @@ -77,6 +77,10 @@ public class Autoscaling { return peak.equals(Load.zero()); } + public boolean isPresent() { + return ! isEmpty(); + } + @Override public boolean equals(Object o) { if ( ! (o instanceof Autoscaling other)) return false; @@ -102,8 +106,12 @@ public class Autoscaling { } public static Autoscaling empty() { + return empty(""); + } + + public static Autoscaling empty(String description) { return new Autoscaling(Status.unavailable, - "", + description, Optional.empty(), Instant.EPOCH, Load.zero(), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java index 264664f91b2..281d9efe51a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java @@ -189,21 +189,28 @@ public class ClusterModel { /** * Returns the ideal load across the nodes of this such that each node will be at ideal load - * if one of the nodes go down. + * if one of the nodes go down. */ public Load idealLoad() { var ideal = new Load(idealCpuLoad(), idealMemoryLoad(), idealDiskLoad()).divide(redundancyAdjustment()); - if (! cluster.bcpGroupInfo().isEmpty()) { + if ( !cluster.bcpGroupInfo().isEmpty() && cluster.bcpGroupInfo().queryRate() > 0) { + // Since we have little local information, use information about query cost in other groups + + Load bcpGroupIdeal = adjustQueryDependentIdealLoadByBcpGroupInfo(ideal); + // Do a weighted sum of the ideal "vote" based on local and bcp group info. // This avoids any discontinuities with a near-zero local query rate. double localInformationWeight = Math.min(1, averageQueryRate().orElse(0) / Math.min(queryRateGivingFullConfidence, cluster.bcpGroupInfo().queryRate())); - Load bcpGroupIdeal = adjustQueryDependentIdealLoadByBcpGroupInfo(ideal); ideal = ideal.multiply(localInformationWeight).add(bcpGroupIdeal.multiply(1 - localInformationWeight)); } return ideal; } + private boolean canRescaleWithinBcpDeadline() { + return scalingDuration().minus(cluster.clusterInfo().bcpDeadline()).isNegative(); + } + public Autoscaling.Metrics metrics() { return new Autoscaling.Metrics(averageQueryRate().orElse(0), growthRateHeadroom(), @@ -214,7 +221,7 @@ public class ClusterModel { public Instant at() { return at;} private OptionalDouble cpuCostPerQuery() { - if (averageQueryRate().isEmpty()) return OptionalDouble.empty(); + if (averageQueryRate().isEmpty() || averageQueryRate().getAsDouble() == 0.0) return OptionalDouble.empty(); // TODO: Query rate should generally be sampled at the time where we see the peak resource usage int fanOut = clusterSpec.type().isContainer() ? 1 : groupSize(); return OptionalDouble.of(peakLoad().cpu() * queryCpuFraction() * fanOut * nodes.not().retired().first().get().resources().vcpu() @@ -224,7 +231,9 @@ public class ClusterModel { private Load adjustQueryDependentIdealLoadByBcpGroupInfo(Load ideal) { double currentClusterTotalVcpuPerGroup = nodes.not().retired().first().get().resources().vcpu() * groupSize(); - double targetQueryRateToHandle = cluster.bcpGroupInfo().queryRate() * cluster.bcpGroupInfo().growthRateHeadroom() * trafficShiftHeadroom(); + double targetQueryRateToHandle = ( canRescaleWithinBcpDeadline() ? averageQueryRate().orElse(0) + : cluster.bcpGroupInfo().queryRate() ) + * cluster.bcpGroupInfo().growthRateHeadroom() * trafficShiftHeadroom(); double neededTotalVcpPerGroup = cluster.bcpGroupInfo().cpuCostPerQuery() * targetQueryRateToHandle / groupCount() + ( 1 - queryCpuFraction()) * idealCpuLoad() * (clusterSpec.type().isContainer() ? 1 : groupSize()); @@ -319,6 +328,7 @@ public class ClusterModel { */ private double trafficShiftHeadroom() { if ( ! zone.environment().isProduction()) return 1; + if (canRescaleWithinBcpDeadline()) return 1; double trafficShiftHeadroom; if (application.status().maxReadShare() == 0) // No traffic fraction data trafficShiftHeadroom = 2.0; // assume we currently get half of the max possible share of traffic diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java index 0be4175c2c1..d694085729f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java @@ -6,10 +6,8 @@ import com.yahoo.vespa.hosted.provision.applications.Cluster; import java.time.Duration; import java.util.List; -import java.util.Optional; import java.util.OptionalDouble; import java.util.function.Predicate; -import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.provision.autoscale.ClusterModel.warmupDuration; @@ -54,8 +52,11 @@ public class ClusterNodesTimeseries { /** Returns the average number of measurements per node */ public double measurementsPerNode() { if (clusterNodes.size() == 0) return 0; - int measurementCount = timeseries.stream().mapToInt(m -> m.size()).sum(); - return (double)measurementCount / clusterNodes.size(); + return (double) totalMeasurementsIn(timeseries) / clusterNodes.size(); + } + + private int totalMeasurementsIn(List<NodeTimeseries> timeseries) { + return timeseries.stream().mapToInt(m -> m.size()).sum(); } /** Returns the number of nodes measured in this */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java index dc0327c9537..8b7a2bafc40 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java @@ -4,6 +4,10 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.collections.ListMap; import com.yahoo.collections.Pair; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.metrics.ContainerMetrics; +import com.yahoo.metrics.HostedNodeAdminMetrics; +import com.yahoo.metrics.SearchNodeMetrics; +import com.yahoo.metrics.StorageMetrics; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; @@ -20,6 +24,9 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import static com.yahoo.metrics.ContainerMetrics.APPLICATION_GENERATION; +import static com.yahoo.metrics.ContainerMetrics.IN_SERVICE; + /** * A response containing metrics for a collection of nodes. * @@ -113,7 +120,7 @@ public class MetricsResponse { cpu { // a node resource @Override - public List<String> metricResponseNames() { return List.of("cpu.util"); } + public List<String> metricResponseNames() { return List.of(HostedNodeAdminMetrics.CPU_UTIL.baseName()); } @Override double computeFinal(ListMap<String, Double> values) { @@ -125,15 +132,16 @@ public class MetricsResponse { @Override public List<String> metricResponseNames() { - return List.of("content.proton.resource_usage.memory.average", "mem.util"); + return List.of(HostedNodeAdminMetrics.MEM_UTIL.baseName(), + SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_MEMORY.average()); } @Override double computeFinal(ListMap<String, Double> values) { - var valueList = values.get("content.proton.resource_usage.memory.average"); // prefer over mem.util + var valueList = values.get(SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_MEMORY.average()); // prefer over mem.util if ( ! valueList.isEmpty()) return valueList.get(0); - valueList = values.get("mem.util"); + valueList = values.get(HostedNodeAdminMetrics.MEM_UTIL.baseName()); if ( ! valueList.isEmpty()) return valueList.get(0) / 100; // % to ratio return 0; @@ -144,15 +152,16 @@ public class MetricsResponse { @Override public List<String> metricResponseNames() { - return List.of("content.proton.resource_usage.disk.average", "disk.util"); + return List.of(HostedNodeAdminMetrics.DISK_UTIL.baseName(), + SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_DISK.average()); } @Override double computeFinal(ListMap<String, Double> values) { - var valueList = values.get("content.proton.resource_usage.disk.average"); // prefer over mem.util + var valueList = values.get(SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_DISK.average()); // prefer over mem.util if ( ! valueList.isEmpty()) return valueList.get(0); - valueList = values.get("disk.util"); + valueList = values.get(HostedNodeAdminMetrics.DISK_UTIL.baseName()); if ( ! valueList.isEmpty()) return valueList.get(0) / 100; // % to ratio return 0; @@ -162,7 +171,9 @@ public class MetricsResponse { generation { // application config generation active on the node @Override - public List<String> metricResponseNames() { return List.of("application_generation"); } + public List<String> metricResponseNames() { + return List.of(APPLICATION_GENERATION.last(), SearchNodeMetrics.CONTENT_PROTON_CONFIG_GENERATION.last()); + } @Override double computeFinal(ListMap<String, Double> values) { @@ -173,7 +184,7 @@ public class MetricsResponse { inService { @Override - public List<String> metricResponseNames() { return List.of("in_service"); } + public List<String> metricResponseNames() { return List.of(IN_SERVICE.last()); } @Override double computeFinal(ListMap<String, Double> values) { @@ -186,8 +197,8 @@ public class MetricsResponse { @Override public List<String> metricResponseNames() { - return List.of("queries.rate", - "content.proton.documentdb.matching.queries.rate"); + return List.of(ContainerMetrics.QUERIES.rate(), + SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_QUERIES.rate()); } }, @@ -195,10 +206,11 @@ public class MetricsResponse { @Override public List<String> metricResponseNames() { - return List.of("feed.http-requests.rate", - "vds.filestor.allthreads.put.count.rate", - "vds.filestor.allthreads.remove.count.rate", - "vds.filestor.allthreads.update.count.rate"); } + return List.of(ContainerMetrics.FEED_HTTP_REQUESTS.rate(), + StorageMetrics.VDS_FILESTOR_ALLTHREADS_PUT_COUNT.rate(), + StorageMetrics.VDS_FILESTOR_ALLTHREADS_REMOVE_COUNT.rate(), + StorageMetrics.VDS_FILESTOR_ALLTHREADS_UPDATE_COUNT.rate()); + } }; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java index c25b0684f5a..f694070a8db 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.node.History; import java.time.Instant; @@ -11,7 +12,6 @@ import java.util.List; import java.util.Optional; import java.util.OptionalDouble; import java.util.function.Predicate; -import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.provision.autoscale.ClusterModel.warmupDuration; @@ -98,7 +98,6 @@ public class NodeTimeseries { } private boolean onAtLeastGeneration(long generation, NodeMetricSnapshot snapshot) { - if (snapshot.generation() < 0) return true; // Content nodes do not yet send generation return snapshot.generation() >= generation; } @@ -112,4 +111,11 @@ public class NodeTimeseries { return up.isPresent() && snapshot.at().isBefore(up.get().at().plus(warmupDuration)); } + String description(long generation, NodeList clusterNodes) { + var node = clusterNodes.node(hostname); + if (node.isEmpty()) return "(no node " + hostname + ")"; + return "gen " + generation + " since " + generationChange(generation) + ", up " + node.get().history().event(History.Event.Type.up) + + ": " + (snapshots.isEmpty() ? "(no snapshots)" : snapshots.get(snapshots.size()-1)); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java index f166e73da77..e228d31384c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java @@ -110,9 +110,18 @@ public class LoadBalancerInstance { return ports; } - /** Prepends the given service IDs, possibly replacing those we have in this. */ + /** Updates this with new data, from a reconfiguration. */ + public LoadBalancerInstance with(Set<Real> reals, ZoneEndpoint settings, Optional<PrivateServiceId> serviceId) { + List<PrivateServiceId> ids = new ArrayList<>(serviceIds); + serviceId.filter(id -> ! ids.contains(id)).ifPresent(ids::add); + return new LoadBalancerInstance(hostname, ipAddress, dnsZone, ports, networks, + reals, settings, ids, + cloudAccount); + } + + /** Prepends the given service IDs, possibly changing the order of those we have in this. */ public LoadBalancerInstance withServiceIds(List<PrivateServiceId> serviceIds) { - List<PrivateServiceId> ids = new ArrayList<>(serviceIds()); + List<PrivateServiceId> ids = new ArrayList<>(serviceIds); for (PrivateServiceId id : this.serviceIds) if ( ! ids.contains(id)) ids.add(id); return new LoadBalancerInstance(hostname, ipAddress, dnsZone, ports, networks, reals, settings, ids, cloudAccount); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java index 82f679bbe34..ceedbcf89c2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java @@ -22,12 +22,13 @@ public interface LoadBalancerService { /** * Configures load balancers for the given specification. Implementations are expected to be idempotent * + * @param instance Load balancer instance to reconfigure * @param spec Load balancer specification * @param force Whether reconfiguration should be forced (e.g. allow configuring an empty set of reals on a * pre-existing load balancer, or removing an unused private endpoint service load balancer). * @return The (re)configured load balancer instance */ - LoadBalancerInstance configure(LoadBalancerSpec spec, boolean force); + LoadBalancerInstance configure(LoadBalancerInstance instance, LoadBalancerSpec spec, boolean force); /** Permanently remove given load balancer */ void remove(LoadBalancer loadBalancer); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java index d35c11783dd..1f354dc7081 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java @@ -68,25 +68,17 @@ public class LoadBalancerServiceMock implements LoadBalancerService { } @Override - public LoadBalancerInstance configure(LoadBalancerSpec spec, boolean force) { - if (throwOnCreate) throw new IllegalStateException("Did not expect a new load balancer to be created"); + public LoadBalancerInstance configure(LoadBalancerInstance instance, LoadBalancerSpec spec, boolean force) { var id = new LoadBalancerId(spec.application(), spec.cluster()); var oldInstance = Objects.requireNonNull(instances.get(id), "expected existing load balancer " + id); if (!force && !oldInstance.reals().isEmpty() && spec.reals().isEmpty()) { throw new IllegalArgumentException("Refusing to remove all reals from load balancer " + id); } - var instance = new LoadBalancerInstance( - Optional.of(DomainName.of("lb-" + spec.application().toShortString() + "-" + spec.cluster().value())), - Optional.empty(), - Optional.of(new DnsZone("zone-id-1")), - Collections.singleton(4443), - ImmutableSet.of("10.2.3.0/24", "10.4.5.0/24"), - spec.reals(), - spec.settings(), - spec.settings().isPrivateEndpoint() ? List.of(PrivateServiceId.of("service")) : List.of(), - spec.cloudAccount()); - instances.put(id, instance); - return instance; + var updated = instance.with(spec.reals(), + spec.settings(), + spec.settings().isPrivateEndpoint() ? Optional.of(PrivateServiceId.of("service")) : Optional.empty()); + instances.put(id, updated); + return updated; } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java index 432b56c7b48..22367f72666 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java @@ -33,8 +33,8 @@ public class SharedLoadBalancerService implements LoadBalancerService { } @Override - public LoadBalancerInstance configure(LoadBalancerSpec spec, boolean force) { - return create(spec); + public LoadBalancerInstance configure(LoadBalancerInstance instance, LoadBalancerSpec spec, boolean force) { + return instance.with(spec.reals(), spec.settings(), Optional.empty()); } private LoadBalancerInstance create(LoadBalancerSpec spec) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 674c20e25f2..69c03dbf6dc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Deployer; @@ -47,27 +48,30 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { @Override protected double maintain() { if ( ! nodeRepository().nodes().isWorking()) return 0.0; - - if ( ! nodeRepository().zone().environment().isAnyOf(Environment.dev, Environment.perf, Environment.prod)) return 1.0; - - activeNodesByApplication().forEach(this::autoscale); - return 1.0; - } - - private void autoscale(ApplicationId application, NodeList applicationNodes) { - try { - nodesByCluster(applicationNodes).forEach((clusterId, clusterNodes) -> autoscale(application, clusterId)); - } - catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Illegal arguments for " + application, e); + if (nodeRepository().zone().environment().isTest()) return 1.0; + + int attempts = 0; + int failures = 0; + for (var applicationNodes : activeNodesByApplication().entrySet()) { + for (var clusterNodes : nodesByCluster(applicationNodes.getValue()).entrySet()) { + attempts++; + if ( ! autoscale(applicationNodes.getKey(), clusterNodes.getKey())) + failures++; + } } + return asSuccessFactor(attempts, failures); } - private void autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId) { + /** + * Autoscales the given cluster. + * + * @return true if an autoscaling decision was made or nothing should be done, false if there was an error + */ + private boolean autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId) { try (var lock = nodeRepository().applications().lock(applicationId)) { Optional<Application> application = nodeRepository().applications().get(applicationId); - if (application.isEmpty()) return; - if (application.get().cluster(clusterId).isEmpty()) return; + if (application.isEmpty()) return true; + if (application.get().cluster(clusterId).isEmpty()) return true; Cluster cluster = application.get().cluster(clusterId).get(); NodeList clusterNodes = nodeRepository().nodes().list(Node.State.active).owner(applicationId).cluster(clusterId); @@ -79,7 +83,7 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { Autoscaling autoscaling = null; if (cluster.target().resources().isEmpty() || current.equals(cluster.target().resources().get())) { autoscaling = autoscaler.autoscale(application.get(), cluster, clusterNodes); - if ( ! autoscaling.isEmpty()) // Ignore empties we'll get from servers recently started + if ( autoscaling.isPresent() || cluster.target().isEmpty()) // Ignore empty from recently started servers cluster = cluster.withTarget(autoscaling); } @@ -94,6 +98,13 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { logAutoscaling(current, autoscaling.resources().get(), applicationId, clusterNodes.not().retired()); } } + return true; + } + catch (ApplicationLockException e) { + return false; + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Illegal arguments for " + applicationId + " cluster " + clusterId, e); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java index c8b736cb25b..603056856e2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java @@ -127,12 +127,12 @@ public class CapacityChecker { for (var host : hosts) { NodeResources hostResources = host.flavor().resources(); int occupiedIps = 0; - Set<String> ipPool = host.ipConfig().pool().asSet(); + Set<String> ipPool = host.ipConfig().pool().ipSet(); for (var child : nodeChildren.get(host)) { hostResources = hostResources.subtract(child.resources().justNumbers()); occupiedIps += child.ipConfig().primary().stream().filter(ipPool::contains).count(); } - availableResources.put(host, new AllocationResources(hostResources, host.ipConfig().pool().asSet().size() - occupiedIps)); + availableResources.put(host, new AllocationResources(hostResources, host.ipConfig().pool().ipSet().size() - occupiedIps)); } return availableResources; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java index 1d5581b511d..83dadddf76c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java @@ -196,8 +196,8 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { Version osVersion = nodeRepository().osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion); List<Integer> provisionIndices = nodeRepository().database().readProvisionIndices(count); List<Node> hosts = new ArrayList<>(); - hostProvisioner.provisionHosts(provisionIndices, NodeType.host, nodeResources, ApplicationId.defaultId(), - osVersion, HostSharing.shared, Optional.empty(), nodeRepository().zone().cloud().account(), + hostProvisioner.provisionHosts(provisionIndices, NodeType.host, nodeResources, ApplicationId.defaultId(), osVersion, + HostSharing.shared, Optional.empty(), Optional.empty(), nodeRepository().zone().cloud().account(), provisionedHosts -> { hosts.addAll(provisionedHosts.stream().map(ProvisionedHost::generateHost).toList()); nodeRepository().nodes().addNodes(hosts, Agent.HostCapacityMaintainer); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java index 86c5a926900..c606ede05d1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java @@ -9,14 +9,17 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; -import com.yahoo.vespa.hosted.provision.provisioning.HostIpConfig; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; import com.yahoo.yolean.Exceptions; import javax.naming.NamingException; import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; /** * Resumes provisioning (requests additional IP addresses, updates DNS when IPs are ready) of hosts in state provisioned @@ -38,13 +41,23 @@ public class HostResumeProvisioner extends NodeRepositoryMaintainer { @Override protected double maintain() { NodeList allNodes = nodeRepository().nodes().list(); + Map<String, Set<Node>> nodesByProvisionedParentHostname = + allNodes.nodeType(NodeType.tenant, NodeType.config, NodeType.controller) + .asList() + .stream() + .filter(node -> node.parentHostname().isPresent()) + .collect(Collectors.groupingBy(node -> node.parentHostname().get(), Collectors.toSet())); + NodeList hosts = allNodes.state(Node.State.provisioned).nodeType(NodeType.host, NodeType.confighost, NodeType.controllerhost); int failures = 0; for (Node host : hosts) { - NodeList children = allNodes.childrenOf(host); - try { - HostIpConfig hostIpConfig = hostProvisioner.provision(host, children.asSet()); - setIpConfig(host, children, hostIpConfig); + Set<Node> children = nodesByProvisionedParentHostname.getOrDefault(host.hostname(), Set.of()); + // This doesn't actually require unallocated lock, but that is much easier than simultaneously holding + // the application locks of the host and all it's children. + try (var lock = nodeRepository().nodes().lockUnallocated()) { + List<Node> updatedNodes = hostProvisioner.provision(host, children); + verifyDns(updatedNodes); + nodeRepository().nodes().write(updatedNodes, lock); } catch (IllegalArgumentException | IllegalStateException e) { log.log(Level.INFO, "Could not provision " + host.hostname() + " with " + children.size() + " children, will retry in " + interval() + ": " + Exceptions.toMessageString(e)); @@ -66,21 +79,13 @@ public class HostResumeProvisioner extends NodeRepositoryMaintainer { return asSuccessFactor(hosts.size(), failures); } - private void setIpConfig(Node host, NodeList children, HostIpConfig hostIpConfig) { - if (hostIpConfig.isEmpty()) return; - NodeList nodes = NodeList.of(host).and(children); + /** Verify DNS configuration of given nodes */ + private void verifyDns(List<Node> nodes) { for (var node : nodes) { - verifyDns(node, hostIpConfig.require(node.hostname())); - } - nodeRepository().nodes().setIpConfig(hostIpConfig); - } - - /** Verify DNS configuration of given node */ - private void verifyDns(Node node, IP.Config ipConfig) { - boolean enclave = node.cloudAccount().isEnclave(nodeRepository().zone()); - for (var ipAddress : ipConfig.primary()) { - IP.verifyDns(node.hostname(), ipAddress, nodeRepository().nameResolver(), !enclave); + boolean enclave = node.cloudAccount().isEnclave(nodeRepository().zone()); + for (var ipAddress : node.ipConfig().primary()) { + IP.verifyDns(node.hostname(), ipAddress, nodeRepository().nameResolver(), !enclave); + } } } - } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index b1beb615bc3..f864ab18920 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -112,7 +112,8 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { try { attempts.add(1); LOG.log(Level.INFO, () -> "Removing reals from inactive load balancer " + lb.id() + ": " + Sets.difference(lb.instance().get().reals(), reals)); - LoadBalancerInstance instance = service.configure(new LoadBalancerSpec(lb.id().application(), lb.id().cluster(), reals, + LoadBalancerInstance instance = service.configure(lb.instance().get(), + new LoadBalancerSpec(lb.id().application(), lb.id().cluster(), reals, lb.instance().get().settings(), lb.instance().get().cloudAccount()), true); db.writeLoadBalancer(lb.with(Optional.of(instance)), lb.state()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index 67c1c7359f7..5af74214648 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -267,13 +267,20 @@ public class MetricsReporter extends NodeRepositoryMaintainer { } private void updateNodeCountMetrics(NodeList nodes) { - Map<State, List<Node>> nodesByState = nodes.nodeType(NodeType.tenant).asList().stream() - .collect(Collectors.groupingBy(Node::state)); + var nodesByState = nodes.nodeType(NodeType.tenant) + .asList().stream() + .collect(Collectors.groupingBy(Node::state)); + + var hostsByState = nodes.nodeType(NodeType.host) + .asList().stream() + .collect(Collectors.groupingBy(Node::state)); // Count per state for (State state : State.values()) { - List<Node> nodesInState = nodesByState.getOrDefault(state, List.of()); - metric.set("hostedVespa." + state.name() + "Hosts", nodesInState.size(), null); + var nodesInState = nodesByState.getOrDefault(state, List.of()); + var hostsInState = hostsByState.getOrDefault(state, List.of()); + metric.set("hostedVespa." + state.name() + "Nodes", nodesInState.size(), null); + metric.set("hostedVespa." + state.name() + "Hosts", hostsInState.size(), null); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java index 94683d463af..781debe26a0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java @@ -42,14 +42,13 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer { @Override protected double maintain() { - return updateActiveNodeDownState(); + return updateNodeHealth(); } /** - * If the node is down (see {@link #allDown}), and there is no "down" history record, we add it. - * Otherwise we remove any "down" history record. + * Update UP and DOWN node records for each node as they change. */ - private double updateActiveNodeDownState() { + private double updateNodeHealth() { var attempts = new MutableInteger(0); var failures = new MutableInteger(0); NodeList activeNodes = nodeRepository().nodes().list(Node.State.active); @@ -59,7 +58,7 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer { // Already correct record, nothing to do boolean isDown = allDown(serviceInstances); - if (isDown == node.get().isDown()) return; + if (isDown == node.get().isDown() && isDown != node.get().isUp()) return; // Lock and update status ApplicationId owner = node.get().allocation().get().owner(); @@ -82,7 +81,7 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer { } /** - * Returns true if the node is considered bad: All monitored services services are down. + * Returns true if the node is considered bad: All monitored services are down. * If a node remains bad for a long time, the NodeFailer will try to fail the node. */ static boolean allDown(List<ServiceInstance> services) { @@ -110,7 +109,7 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer { /** Clear down record for node, if any */ private void recordAsUp(Node node, Mutex lock) { - if (!node.isDown()) return; // already up: Don't change down timestamp + if (node.isUp()) return; // already up: Don't change up timestamp nodeRepository().nodes().write(node.upAt(clock().instant(), Agent.NodeHealthTracker), lock); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Address.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Address.java new file mode 100644 index 00000000000..d7ef2228960 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Address.java @@ -0,0 +1,17 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.node; + +import java.util.Objects; + +/** + * Address info about a container that might run on a host. + * + * @author hakon + */ +public record Address(String hostname) { + public Address { + Objects.requireNonNull(hostname, "hostname cannot be null"); + if (hostname.isEmpty()) + throw new IllegalArgumentException("hostname cannot be empty"); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java index 708d84ba655..b2becc7ecfd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node; +import com.google.common.collect.ImmutableSet; import com.google.common.net.InetAddresses; import com.google.common.primitives.UnsignedBytes; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -31,7 +31,7 @@ import static com.yahoo.config.provision.NodeType.proxyhost; * * @author mpolden */ -public record IP() { +public class IP { /** Comparator for sorting IP addresses by their natural order */ public static final Comparator<InetAddress> NATURAL_ORDER = (ip1, ip2) -> { @@ -59,26 +59,31 @@ public record IP() { }; /** IP configuration of a node */ - public record Config(Set<String> primary, Pool pool) { + public static class Config { public static final Config EMPTY = Config.ofEmptyPool(Set.of()); + private final Set<String> primary; + private final Pool pool; + public static Config ofEmptyPool(Set<String> primary) { - return new Config(primary, Pool.EMPTY); + return new Config(primary, Set.of(), List.of()); } - public static Config of(Set<String> primary, Set<String> ipPool, List<HostName> hostnames) { - return new Config(primary, new Pool(IpAddresses.of(ipPool), hostnames)); + public static Config of(Set<String> primary, Set<String> ipPool, List<Address> addressPool) { + return new Config(primary, ipPool, addressPool); } - public static Config of(Set<String> primary, Set<String> pool) { - return of(primary, pool, List.of()); + /** LEGACY TEST CONSTRUCTOR - use of() variants and/or the with- methods. */ + public Config(Set<String> primary, Set<String> pool) { + this(primary, pool, List.of()); } /** DO NOT USE: Public for NodeSerializer. */ - public Config(Set<String> primary, Pool pool) { - this.primary = Collections.unmodifiableSet(new LinkedHashSet<>(Objects.requireNonNull(primary, "primary must be non-null"))); - this.pool = Objects.requireNonNull(pool, "pool must be non-null"); + public Config(Set<String> primary, Set<String> pool, List<Address> addresses) { + this.primary = ImmutableSet.copyOf(Objects.requireNonNull(primary, "primary must be non-null")); + this.pool = Pool.of(Objects.requireNonNull(pool, "pool must be non-null"), + Objects.requireNonNull(addresses, "addresses must be non-null")); } /** The primary addresses of this. These addresses are used when communicating with the node itself */ @@ -93,12 +98,31 @@ public record IP() { /** Returns a copy of this with pool set to given value */ public Config withPool(Pool pool) { - return new Config(primary, pool); + return new Config(primary, pool.ipSet(), pool.getAddressList()); } /** Returns a copy of this with pool set to given value */ public Config withPrimary(Set<String> primary) { - return new Config(primary, pool); + return new Config(primary, pool.ipSet(), pool.getAddressList()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Config config = (Config) o; + return primary.equals(config.primary) && + pool.equals(config.pool); + } + + @Override + public int hashCode() { + return Objects.hash(primary, pool); + } + + @Override + public String toString() { + return String.format("ip config primary=%s pool=%s", primary, pool.ipSet()); } /** @@ -116,8 +140,8 @@ public record IP() { var addresses = new HashSet<>(node.ipConfig().primary()); var otherAddresses = new HashSet<>(other.ipConfig().primary()); if (node.type().isHost()) { // Addresses of a host can never overlap with any other nodes - addresses.addAll(node.ipConfig().pool().asSet()); - otherAddresses.addAll(other.ipConfig().pool().asSet()); + addresses.addAll(node.ipConfig().pool().ipSet()); + otherAddresses.addAll(other.ipConfig().pool().ipSet()); } otherAddresses.retainAll(addresses); if (!otherAddresses.isEmpty()) @@ -134,12 +158,12 @@ public record IP() { if (node.parentHostname().isPresent() == existingNode.parentHostname().isPresent()) return false; // Not a parent-child node if (node.parentHostname().isEmpty()) return canAssignIpOf(node, existingNode); if (!node.parentHostname().get().equals(existingNode.hostname())) return false; // Wrong host - return switch (node.type()) { - case proxy -> existingNode.type() == proxyhost; - case config -> existingNode.type() == confighost; - case controller -> existingNode.type() == controllerhost; - default -> false; - }; + switch (node.type()) { + case proxy: return existingNode.type() == proxyhost; + case config: return existingNode.type() == confighost; + case controller: return existingNode.type() == controllerhost; + } + return false; } public static Node verify(Node node, LockedNodeList allNodes) { @@ -149,13 +173,25 @@ public record IP() { } /** A list of IP addresses and their protocol */ - record IpAddresses(Set<String> addresses, Protocol protocol) { + public static class IpAddresses { - public IpAddresses(Set<String> addresses, Protocol protocol) { - this.addresses = Collections.unmodifiableSet(new LinkedHashSet<>(Objects.requireNonNull(addresses, "addresses must be non-null"))); + private final Set<String> ipAddresses; + private final Protocol protocol; + + private IpAddresses(Set<String> ipAddresses, Protocol protocol) { + this.ipAddresses = ImmutableSet.copyOf(Objects.requireNonNull(ipAddresses, "addresses must be non-null")); this.protocol = Objects.requireNonNull(protocol, "type must be non-null"); } + public Set<String> asSet() { + return ipAddresses; + } + + /** The protocol of addresses in this */ + public Protocol protocol() { + return protocol; + } + /** Create addresses of the given set */ private static IpAddresses of(Set<String> addresses) { long ipv6AddrCount = addresses.stream().filter(IP::isV6).count(); @@ -180,7 +216,6 @@ public record IP() { } public enum Protocol { - dualStack("dual-stack"), ipv4("IPv4-only"), ipv6("IPv6-only"); @@ -189,8 +224,21 @@ public record IP() { Protocol(String description) { this.description = description; } + public String getDescription() { return description; } } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + IpAddresses that = (IpAddresses) o; + return ipAddresses.equals(that.ipAddresses) && protocol == that.protocol; + } + + @Override + public int hashCode() { + return Objects.hash(ipAddresses, protocol); + } } /** @@ -198,23 +246,25 @@ public record IP() { * * Addresses in this are available for use by Linux containers. */ - public record Pool(IpAddresses ipAddresses, List<HostName> hostnames) { + public static class Pool { - public static final Pool EMPTY = new Pool(IpAddresses.of(Set.of()), List.of()); + private final IpAddresses ipAddresses; + private final List<Address> addresses; + + /** Creates an empty pool. */ + public static Pool of() { + return of(Set.of(), List.of()); + } /** Create a new pool containing given ipAddresses */ - public static Pool of(Set<String> ipAddresses, List<HostName> hostnames) { + public static Pool of(Set<String> ipAddresses, List<Address> addresses) { IpAddresses ips = IpAddresses.of(ipAddresses); - return new Pool(ips, hostnames); + return new Pool(ips, addresses); } - public Pool(IpAddresses ipAddresses, List<HostName> hostnames) { + private Pool(IpAddresses ipAddresses, List<Address> addresses) { this.ipAddresses = Objects.requireNonNull(ipAddresses, "ipAddresses must be non-null"); - this.hostnames = List.copyOf(Objects.requireNonNull(hostnames, "hostnames must be non-null")); - } - - public Set<String> asSet() { - return ipAddresses.addresses; + this.addresses = Objects.requireNonNull(addresses, "addresses must be non-null"); } /** @@ -224,15 +274,16 @@ public record IP() { * @return an allocation from the pool, if any can be made */ public Optional<Allocation> findAllocation(LockedNodeList nodes, NameResolver resolver, boolean hasPtr) { - if (ipAddresses.addresses.isEmpty()) { + if (ipAddresses.asSet().isEmpty()) { // IP addresses have not yet been resolved and should be done later. - return findUnusedHostnames(nodes).map(Allocation::ofHostname) - .findFirst(); + return findUnusedAddressStream(nodes) + .map(Allocation::ofAddress) + .findFirst(); } if (!hasPtr) { // Without PTR records (reverse IP mapping): Ensure only forward resolving from hostnames. - return findUnusedHostnames(nodes).findFirst().map(address -> Allocation.fromHostname(address, resolver, ipAddresses.protocol)); + return findUnusedAddressStream(nodes).findFirst().map(address -> Allocation.fromAddress(address, resolver, ipAddresses.protocol)); } if (ipAddresses.protocol == IpAddresses.Protocol.ipv4) { @@ -262,34 +313,64 @@ public record IP() { * @param nodes a list of all nodes in the repository */ public Set<String> findUnusedIpAddresses(NodeList nodes) { - Set<String> unusedAddresses = new LinkedHashSet<>(asSet()); - nodes.matching(node -> node.ipConfig().primary().stream().anyMatch(ip -> asSet().contains(ip))) + var unusedAddresses = new LinkedHashSet<>(ipSet()); + nodes.matching(node -> node.ipConfig().primary().stream().anyMatch(ip -> ipSet().contains(ip))) .forEach(node -> unusedAddresses.removeAll(node.ipConfig().primary())); return Collections.unmodifiableSet(unusedAddresses); } - private Stream<HostName> findUnusedHostnames(NodeList nodes) { - Set<String> usedHostnames = nodes.stream().map(Node::hostname).collect(Collectors.toSet()); - return hostnames.stream().filter(hostname -> !usedHostnames.contains(hostname.value())); + private Stream<Address> findUnusedAddressStream(NodeList nodes) { + Set<String> hostnames = nodes.stream().map(Node::hostname).collect(Collectors.toSet()); + return addresses.stream().filter(address -> !hostnames.contains(address.hostname())); + } + + public IpAddresses.Protocol getProtocol() { + return ipAddresses.protocol; + } + + /** Returns the IP addresses in this pool as a set */ + public Set<String> ipSet() { + return ipAddresses.asSet(); + } + + public List<Address> getAddressList() { + return addresses; } public Pool withIpAddresses(Set<String> ipAddresses) { - return Pool.of(ipAddresses, hostnames); + return Pool.of(ipAddresses, addresses); } - public Pool withHostnames(List<HostName> hostnames) { - return Pool.of(ipAddresses.addresses, hostnames); + public Pool withAddresses(List<Address> addresses) { + return Pool.of(ipAddresses.ipAddresses, addresses); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Pool pool = (Pool) o; + return ipAddresses.equals(pool.ipAddresses) && addresses.equals(pool.addresses); + } + + @Override + public int hashCode() { + return Objects.hash(ipAddresses, addresses); } } /** An address allocation from a pool */ - public record Allocation(String hostname, Optional<String> ipv4Address, Optional<String> ipv6Address) { + public static class Allocation { + + private final String hostname; + private final Optional<String> ipv4Address; + private final Optional<String> ipv6Address; - public Allocation { - Objects.requireNonNull(hostname, "hostname must be non-null"); - Objects.requireNonNull(ipv4Address, "ipv4Address must be non-null"); - Objects.requireNonNull(ipv6Address, "ipv6Address must be non-null"); + private Allocation(String hostname, Optional<String> ipv4Address, Optional<String> ipv6Address) { + this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); + this.ipv4Address = Objects.requireNonNull(ipv4Address, "ipv4Address must be non-null"); + this.ipv6Address = Objects.requireNonNull(ipv6Address, "ipv6Address must be non-null"); } /** @@ -346,22 +427,22 @@ public record IP() { return new Allocation(hostname4, Optional.of(addresses.get(0)), Optional.empty()); } - private static Allocation fromHostname(HostName hostname, NameResolver resolver, IpAddresses.Protocol protocol) { + private static Allocation fromAddress(Address address, NameResolver resolver, IpAddresses.Protocol protocol) { // Resolve both A and AAAA to verify they match the protocol and to avoid surprises later on. - Optional<String> ipv4Address = resolveOptional(hostname.value(), resolver, RecordType.A); + Optional<String> ipv4Address = resolveOptional(address.hostname(), resolver, RecordType.A); if (protocol != IpAddresses.Protocol.ipv6 && ipv4Address.isEmpty()) - throw new IllegalArgumentException(protocol.description + " hostname " + hostname.value() + " did not resolve to an IPv4 address"); + throw new IllegalArgumentException(protocol.description + " hostname " + address.hostname() + " did not resolve to an IPv4 address"); if (protocol == IpAddresses.Protocol.ipv6 && ipv4Address.isPresent()) - throw new IllegalArgumentException(protocol.description + " hostname " + hostname.value() + " has an IPv4 address: " + ipv4Address.get()); + throw new IllegalArgumentException(protocol.description + " hostname " + address.hostname() + " has an IPv4 address: " + ipv4Address.get()); - Optional<String> ipv6Address = resolveOptional(hostname.value(), resolver, RecordType.AAAA); + Optional<String> ipv6Address = resolveOptional(address.hostname(), resolver, RecordType.AAAA); if (protocol != IpAddresses.Protocol.ipv4 && ipv6Address.isEmpty()) - throw new IllegalArgumentException(protocol.description + " hostname " + hostname.value() + " did not resolve to an IPv6 address"); + throw new IllegalArgumentException(protocol.description + " hostname " + address.hostname() + " did not resolve to an IPv6 address"); if (protocol == IpAddresses.Protocol.ipv4 && ipv6Address.isPresent()) - throw new IllegalArgumentException(protocol.description + " hostname " + hostname.value() + " has an IPv6 address: " + ipv6Address.get()); + throw new IllegalArgumentException(protocol.description + " hostname " + address.hostname() + " has an IPv6 address: " + ipv6Address.get()); - return new Allocation(hostname.value(), ipv4Address, ipv6Address); + return new Allocation(address.hostname(), ipv4Address, ipv6Address); } private static Optional<String> resolveOptional(String hostname, NameResolver resolver, RecordType recordType) { @@ -373,14 +454,37 @@ public record IP() { }; } - private static Allocation ofHostname(HostName hostName) { - return new Allocation(hostName.value(), Optional.empty(), Optional.empty()); + private static Allocation ofAddress(Address address) { + return new Allocation(address.hostname(), Optional.empty(), Optional.empty()); + } + + /** Hostname pointing to the IP addresses in this */ + public String hostname() { + return hostname; + } + + /** IPv4 address of this allocation */ + public Optional<String> ipv4Address() { + return ipv4Address; + } + + /** IPv6 address of this allocation */ + public Optional<String> ipv6Address() { + return ipv6Address; } /** All IP addresses in this */ public Set<String> addresses() { - return Stream.concat(ipv4Address.stream(), ipv6Address.stream()) - .collect(Collectors.toUnmodifiableSet()); + ImmutableSet.Builder<String> builder = ImmutableSet.builder(); + ipv4Address.ifPresent(builder::add); + ipv6Address.ifPresent(builder::add); + return builder.build(); + } + + @Override + public String toString() { + return String.format("Address allocation [hostname=%s, IPv4=%s, IPv6=%s]", + hostname, ipv4Address.orElse("<none>"), ipv6Address.orElse("<none>")); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index 9134c376f38..bb3d288e555 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.hosted.provision.applications.Applications; import com.yahoo.vespa.hosted.provision.maintenance.NodeFailer; import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; import com.yahoo.vespa.hosted.provision.persistence.CuratorDb; -import com.yahoo.vespa.hosted.provision.provisioning.HostIpConfig; import com.yahoo.vespa.orchestrator.HostNameNotFoundException; import com.yahoo.vespa.orchestrator.Orchestrator; @@ -110,7 +109,7 @@ public class Nodes { */ public boolean isWorking() { NodeList activeNodes = list(Node.State.active); - if (activeNodes.size() <= 5) return true; // Not enough data to decide + if (activeNodes.size() < 20) return true; // Not enough data to decide NodeList downNodes = activeNodes.down(); return ! ( (double)downNodes.size() / (double)activeNodes.size() > 0.2 ); } @@ -126,7 +125,7 @@ public class Nodes { illegal("Cannot add " + node + ": Child nodes need to be allocated"); Optional<Node> existing = node(node.hostname()); if (existing.isPresent()) - illegal("Cannot add " + node + ": A node with this name already exists"); + throw new IllegalStateException("Cannot add " + node + ": A node with this name already exists"); } return db.addNodesInState(nodes.asList(), Node.State.reserved, Agent.system); } @@ -152,7 +151,7 @@ public class Nodes { Optional<Node> existing = node(node.hostname()); if (existing.isPresent()) { if (existing.get().state() != Node.State.deprovisioned) - illegal("Cannot add " + node + ": A node with this name already exists"); + throw new IllegalStateException("Cannot add " + node + ": A node with this name already exists"); node = node.with(existing.get().history()); node = node.with(existing.get().reports()); node = node.with(node.status().withFailCount(existing.get().status().failCount())); @@ -355,15 +354,6 @@ public class Nodes { } } - /** Update IP config for nodes in given config */ - public void setIpConfig(HostIpConfig hostIpConfig) { - Predicate<Node> nodeInConfig = (node) -> hostIpConfig.contains(node.hostname()); - performOn(nodeInConfig, (node, lock) -> { - IP.Config ipConfig = hostIpConfig.require(node.hostname()); - return write(node.with(ipConfig), lock); - }); - } - /** * Parks this node and returns it in its new state. * diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java index 38675ba3758..1e378c80f90 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterResources; @@ -20,6 +21,7 @@ import com.yahoo.vespa.hosted.provision.autoscale.Load; import java.io.IOException; import java.io.UncheckedIOException; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; @@ -54,6 +56,8 @@ public class ApplicationSerializer { private static final String groupSizeKey = "groupSize"; private static final String requiredKey = "required"; private static final String suggestedKey = "suggested"; + private static final String clusterInfoKey = "clusterInfo"; + private static final String bcpDeadlineKey = "bcpDeadline"; private static final String bcpGroupInfoKey = "bcpGroupInfo"; private static final String queryRateKey = "queryRateKey"; private static final String growthRateHeadroomKey = "growthRateHeadroomKey"; @@ -134,6 +138,8 @@ public class ApplicationSerializer { clusterObject.setBool(requiredKey, cluster.required()); toSlime(cluster.suggested(), clusterObject.setObject(suggestedKey)); toSlime(cluster.target(), clusterObject.setObject(targetKey)); + if (! cluster.clusterInfo().isEmpty()) + toSlime(cluster.clusterInfo(), clusterObject.setObject(clusterInfoKey)); if (! cluster.bcpGroupInfo().isEmpty()) toSlime(cluster.bcpGroupInfo(), clusterObject.setObject(bcpGroupInfoKey)); scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray(scalingEventsKey)); @@ -148,6 +154,7 @@ public class ApplicationSerializer { clusterObject.field(requiredKey).asBool(), autoscalingFromSlime(clusterObject.field(suggestedKey)), autoscalingFromSlime(clusterObject.field(targetKey)), + clusterInfoFromSlime(clusterObject.field(clusterInfoKey)), bcpGroupInfoFromSlime(clusterObject.field(bcpGroupInfoKey)), scalingEventsFromSlime(clusterObject.field(scalingEventsKey))); } @@ -225,8 +232,18 @@ public class ApplicationSerializer { metricsFromSlime(autoscalingObject.field(metricsKey))); } + private static void toSlime(ClusterInfo clusterInfo, Cursor clusterInfoObject) { + clusterInfoObject.setLong(bcpDeadlineKey, clusterInfo.bcpDeadline().toMinutes()); + } + + private static ClusterInfo clusterInfoFromSlime(Inspector clusterInfoObject) { + if ( ! clusterInfoObject.valid()) return ClusterInfo.empty(); + ClusterInfo.Builder builder = new ClusterInfo.Builder(); + builder.bcpDeadline(Duration.ofMinutes(clusterInfoObject.field(bcpDeadlineKey).asLong())); + return builder.build(); + } + private static void toSlime(BcpGroupInfo bcpGroupInfo, Cursor bcpGroupInfoObject) { - if (bcpGroupInfo.isEmpty()) return; bcpGroupInfoObject.setDouble(queryRateKey, bcpGroupInfo.queryRate()); bcpGroupInfoObject.setDouble(growthRateHeadroomKey, bcpGroupInfo.growthRateHeadroom()); bcpGroupInfoObject.setDouble(cpuCostPerQueryKey, bcpGroupInfo.cpuCostPerQuery()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ArchiveUriSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ArchiveUriSerializer.java new file mode 100644 index 00000000000..3bf37816dd4 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ArchiveUriSerializer.java @@ -0,0 +1,56 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.TenantName; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.ObjectTraverser; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.hosted.provision.archive.ArchiveUris; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Serializer for archive URIs that are set per tenant and per account. + * + * @author freva + */ +public class ArchiveUriSerializer { + + private ArchiveUriSerializer() {} + + public static byte[] toJson(ArchiveUris archiveUris) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + + Cursor tenantObject = root.setObject("tenant"); + archiveUris.tenantArchiveUris().forEach((tenant, uri) -> tenantObject.setString(tenant.value(), uri)); + + Cursor accountObject = root.setObject("account"); + archiveUris.accountArchiveUris().forEach((account, uri) -> accountObject.setString(account.value(), uri)); + + try { + return SlimeUtils.toJsonBytes(slime); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static ArchiveUris fromJson(byte[] data) { + Inspector inspector = SlimeUtils.jsonToSlime(data).get(); + + Map<TenantName, String> tenantArchiveUris = new HashMap<>(); + inspector.field("tenant").traverse((ObjectTraverser) (key, value) -> tenantArchiveUris.put(TenantName.from(key), value.asString())); + + Map<CloudAccount, String> accountArchiveUris = new HashMap<>(); + inspector.field("account").traverse((ObjectTraverser) (key, value) -> accountArchiveUris.put(CloudAccount.from(key), value.asString())); + + return new ArchiveUris(tenantArchiveUris, accountArchiveUris); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java index c25dfe9f1e2..c1ab8489f40 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java @@ -10,7 +10,6 @@ import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.TenantName; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; @@ -21,6 +20,7 @@ import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.archive.ArchiveUris; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -70,7 +70,7 @@ public class CuratorDb { private static final Path infrastructureVersionsPath = root.append("infrastructureVersions"); private static final Path osVersionsPath = root.append("osVersions"); private static final Path firmwareCheckPath = root.append("firmwareCheck"); - private static final Path archiveUrisPath = root.append("archiveUris"); + private static final Path archiveUrisPath = root.append("archiveUri"); private static final Duration defaultLockTimeout = Duration.ofMinutes(1); @@ -375,16 +375,16 @@ public class CuratorDb { // Archive URIs ----------------------------------------------------------- - public void writeArchiveUris(Map<TenantName, String> archiveUris) { - byte[] data = TenantArchiveUriSerializer.toJson(archiveUris); + public void writeArchiveUris(ArchiveUris archiveUris) { + byte[] data = ArchiveUriSerializer.toJson(archiveUris); NestedTransaction transaction = new NestedTransaction(); CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); curatorTransaction.add(CuratorOperations.setData(archiveUrisPath.getAbsolute(), data)); transaction.commit(); } - public Map<TenantName, String> readArchiveUris() { - return read(archiveUrisPath, TenantArchiveUriSerializer::fromJson).orElseGet(Map::of); + public ArchiveUris readArchiveUris() { + return read(archiveUrisPath, ArchiveUriSerializer::fromJson).orElseGet(() -> new ArchiveUris(Map.of(), Map.of())); } public Lock lockArchiveUris() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 23ea14da4cc..39cccafb8ef 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -14,7 +14,6 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; @@ -29,6 +28,7 @@ import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.Generation; @@ -167,8 +167,8 @@ public class NodeSerializer { object.setString(hostnameKey, node.hostname()); object.setString(stateKey, toString(node.state())); toSlime(node.ipConfig().primary(), object.setArray(ipAddressesKey)); - toSlime(node.ipConfig().pool().asSet(), object.setArray(ipAddressPoolKey)); - toSlime(node.ipConfig().pool().hostnames(), object); + toSlime(node.ipConfig().pool().ipSet(), object.setArray(ipAddressPoolKey)); + toSlime(node.ipConfig().pool().getAddressList(), object); object.setString(idKey, node.id()); node.parentHostname().ifPresent(hostname -> object.setString(parentHostnameKey, hostname)); toSlime(node.flavor(), object); @@ -247,11 +247,11 @@ public class NodeSerializer { ipAddresses.stream().map(IP::parse).sorted(IP.NATURAL_ORDER).map(IP::asString).forEach(array::addString); } - private void toSlime(List<HostName> hostnames, Cursor object) { - if (hostnames.isEmpty()) return; - Cursor containersArray = object.setArray(containersKey); - hostnames.forEach(hostname -> { - containersArray.addObject().setString(containerHostnameKey, hostname.value()); + private void toSlime(List<Address> addresses, Cursor object) { + if (addresses.isEmpty()) return; + Cursor addressCursor = object.setArray(containersKey); + addresses.forEach(address -> { + addressCursor.addObject().setString(containerHostnameKey, address.hostname()); }); } @@ -277,9 +277,9 @@ public class NodeSerializer { private Node nodeFromSlime(Inspector object) { Flavor flavor = flavorFromSlime(object); return new Node(object.field(idKey).asString(), - IP.Config.of(ipAddressesFromSlime(object, ipAddressesKey), - ipAddressesFromSlime(object, ipAddressPoolKey), - hostnamesFromSlime(object)), + new IP.Config(ipAddressesFromSlime(object, ipAddressesKey), + ipAddressesFromSlime(object, ipAddressPoolKey), + addressesFromSlime(object)), object.field(hostnameKey).asString(), SlimeUtils.optionalString(object.field(parentHostnameKey)), flavor, @@ -394,9 +394,9 @@ public class NodeSerializer { return ipAddresses.build(); } - private List<HostName> hostnamesFromSlime(Inspector object) { + private List<Address> addressesFromSlime(Inspector object) { return SlimeUtils.entriesStream(object.field(containersKey)) - .map(elem -> HostName.of(elem.field(containerHostnameKey).asString())) + .map(elem -> new Address(elem.field(containerHostnameKey).asString())) .toList(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/TenantArchiveUriSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/TenantArchiveUriSerializer.java deleted file mode 100644 index d381d25704a..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/TenantArchiveUriSerializer.java +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.persistence; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.ObjectTraverser; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.Map; -import java.util.TreeMap; - -/** - * Serializer for archive URIs that are set per tenant. - * - * @author freva - */ -public class TenantArchiveUriSerializer { - - private TenantArchiveUriSerializer() {} - - public static byte[] toJson(Map<TenantName, String> archiveUrisByTenantName) { - Slime slime = new Slime(); - Cursor object = slime.setObject(); - archiveUrisByTenantName.forEach((tenantName, archiveUri) -> - object.setString(tenantName.value(), archiveUri)); - try { - return SlimeUtils.toJsonBytes(slime); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public static Map<TenantName, String> fromJson(byte[] data) { - Map<TenantName, String> archiveUrisByTenantName = new TreeMap<>(); // Use TreeMap to sort by tenant name - Inspector inspector = SlimeUtils.jsonToSlime(data).get(); - inspector.traverse((ObjectTraverser) (key, value) -> - archiveUrisByTenantName.put(TenantName.from(key), value.asString())); - return archiveUrisByTenantName; - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ArchiveUris.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ArchiveUris.java deleted file mode 100644 index 285365a9c6d..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ArchiveUris.java +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.provisioning; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.Zone; -import com.yahoo.lang.CachedSupplier; -import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.node.Allocation; -import com.yahoo.vespa.hosted.provision.persistence.CuratorDb; - -import java.time.Duration; -import java.util.Map; -import java.util.Optional; -import java.util.TreeMap; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.regex.Pattern; - -/** - * Thread safe class to get and set archive URI for given tenants. Archive URIs are stored in ZooKeeper so that - * nodes within the same tenant have the same archive URI from all the config servers. - * - * @author freva - */ -public class ArchiveUris { - - private static final Logger log = Logger.getLogger(ArchiveUris.class.getName()); - private static final Pattern validUriPattern = Pattern.compile("[a-z0-9]+://(?:(?:[a-z0-9]+(?:[-_][a-z0-9.]+)*)+/)+"); - private static final Duration cacheTtl = Duration.ofMinutes(1); - - private final CuratorDb db; - private final CachedSupplier<Map<TenantName, String>> archiveUris; - private final Zone zone; - - public ArchiveUris(CuratorDb db, Zone zone) { - this.db = db; - this.archiveUris = new CachedSupplier<>(db::readArchiveUris, cacheTtl); - this.zone = zone; - } - - /** Returns the current archive URI for each tenant */ - public Map<TenantName, String> getArchiveUris() { - return archiveUris.get(); - } - - /** Returns the archive URI to use for given tenant */ - private Optional<String> archiveUriFor(TenantName tenant) { - return Optional.ofNullable(archiveUris.get().get(tenant)); - } - - /** Returns the archive URI to use for given node */ - public Optional<String> archiveUriFor(Node node) { - if (node.cloudAccount().isEnclave(zone)) return Optional.empty(); // TODO (freva): Implement for exclave - - return node.allocation().map(Allocation::owner) - .flatMap(app -> archiveUriFor(app.tenant()) - .map(uri -> { - StringBuilder sb = new StringBuilder(100).append(uri) - .append(app.application().value()).append('/') - .append(app.instance().value()).append('/') - .append(node.allocation().get().membership().cluster().id().value()).append('/'); - - for (char c: node.hostname().toCharArray()) { - if (c == '.') break; - sb.append(c); - } - - return sb.append('/').toString(); - })); - } - - /** Set (or remove, if archiveURI is empty) archive URI to use for given tenant */ - public void setArchiveUri(TenantName tenant, Optional<String> archiveUri) { - try (Lock lock = db.lockArchiveUris()) { - Map<TenantName, String> archiveUris = new TreeMap<>(db.readArchiveUris()); - if (Optional.ofNullable(archiveUris.get(tenant)).equals(archiveUri)) return; // No change - - archiveUri.map(ArchiveUris::normalizeUri).ifPresentOrElse(uri -> archiveUris.put(tenant, uri), - () -> archiveUris.remove(tenant)); - db.writeArchiveUris(archiveUris); - this.archiveUris.invalidate(); // Throw away current cache - log.log(Level.FINE, () -> archiveUri.map(s -> "Set archive URI for " + tenant + " to " + s) - .orElseGet(() -> "Remove archive URI for " + tenant)); - } - } - - static String normalizeUri(String uri) { - if (!uri.endsWith("/")) uri = uri + "/"; - if (!validUriPattern.matcher(uri).matches()) - throw new IllegalArgumentException("Invalid archive URI: " + uri); - return uri; - } -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index 8af1df93bde..5732e94956a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -115,10 +115,12 @@ public class CapacityPolicies { if (nodeRepository.exclusiveAllocation(clusterSpec)) { return versioned(clusterSpec, Map.of(new Version(0), smallestExclusiveResources())); } - // TODO (hmusum): Go back to 1.14 Gb memory when bug in resource limits for admin nodes - // has been fixed + + // 1.32 fits floor(8/1.32) = 6 cluster controllers on each 8Gb host, and each will have + // 1.32-(0.7+0.6)*(1.32/8) = 1.1 Gb real memory given current taxes. return versioned(clusterSpec, Map.of(new Version(0), new NodeResources(0.25, 1.14, 10, 0.3), - new Version(8, 127, 11), new NodeResources(0.25, 1.5, 10, 0.3))); + new Version(8, 127, 11), new NodeResources(0.25, 1.5, 10, 0.3), + new Version(8, 129, 4), new NodeResources(0.25, 1.32, 10, 0.3))); } private Architecture adminClusterArchitecture(ApplicationId instance) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 5c2ca58a6b7..691f88a9be3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -115,7 +115,7 @@ public class GroupPreparer { try { hostProvisioner.get().provisionHosts( allocation.provisionIndices(deficit.count()), hostType, deficit.resources(), application, - osVersion, sharing, Optional.of(cluster.type()), requestedNodes.cloudAccount(), + osVersion, sharing, Optional.of(cluster.type()), Optional.of(cluster.id()), requestedNodes.cloudAccount(), provisionedHostsConsumer); } catch (NodeAllocationException e) { // Mark the nodes that were written to ZK in the consumer for deprovisioning. While these hosts do diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostIpConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostIpConfig.java deleted file mode 100644 index 891251fc892..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostIpConfig.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.provisioning; - -import com.yahoo.vespa.hosted.provision.node.IP; - -import java.util.Map; -import java.util.Objects; - -/** - * IP config of a host and its children. - * - * @author mpolden - */ -public record HostIpConfig(Map<String, IP.Config> ipConfigByHostname) { - - public static final HostIpConfig EMPTY = new HostIpConfig(Map.of()); - - public HostIpConfig(Map<String, IP.Config> ipConfigByHostname) { - this.ipConfigByHostname = Map.copyOf(Objects.requireNonNull(ipConfigByHostname)); - } - - public Map<String, IP.Config> asMap() { - return ipConfigByHostname; - } - - public boolean contains(String hostname) { - return ipConfigByHostname.containsKey(hostname); - } - - public IP.Config require(String hostname) { - IP.Config ipConfig = this.ipConfigByHostname.get(hostname); - if (ipConfig == null) throw new IllegalArgumentException("No IP config exists for node '" + hostname + "'"); - return ipConfig; - } - - public boolean isEmpty() { - return this.equals(EMPTY); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java index 3144f42a92c..f07185cbe60 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java @@ -10,6 +10,7 @@ import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; import java.util.List; import java.util.Optional; @@ -49,6 +50,8 @@ public interface HostProvisioner { * @param sharing puts requirements on sharing or exclusivity of the host to be provisioned. * @param clusterType the cluster we are provisioning for, or empty if we are provisioning hosts * to be shared by multiple cluster nodes + * @param clusterId the id of the cluster we are provisioning for, or empty if we are provisioning hosts + * to be shared by multiple cluster nodes * @param cloudAccount the cloud account to use * @param provisionedHostConsumer consumer of {@link ProvisionedHost}s describing the provisioned nodes, * the {@link Node} returned from {@link ProvisionedHost#generateHost()} must be @@ -63,6 +66,7 @@ public interface HostProvisioner { Version osVersion, HostSharing sharing, Optional<ClusterSpec.Type> clusterType, + Optional<ClusterSpec.Id> clusterId, CloudAccount cloudAccount, Consumer<List<ProvisionedHost>> provisionedHostConsumer) throws NodeAllocationException; @@ -71,11 +75,12 @@ public interface HostProvisioner { * * @param host the host to provision * @param children list of all the nodes that run on the given host - * @return IP config for the provisioned host and its children + * @return a subset of {@code host} and {@code children} where the values have been modified and should + * be written back to node-repository. * @throws FatalProvisioningException if the provisioning has irrecoverably failed and the input nodes * should be deleted from node-repo. */ - HostIpConfig provision(Node host, Set<Node> children) throws FatalProvisioningException; + List<Node> provision(Node host, Set<Node> children) throws FatalProvisioningException; /** * Deprovisions a given host and resources associated with it and its children (such as DNS entries). diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 494d50f6a45..cc63ab28a70 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -259,7 +259,7 @@ public class LoadBalancerProvisioner { return Optional.empty(); // Will cause activation to fail, but lets us proceed with more preparations. } - /** Provision or reconfigure a load balancer instance, if necessary */ + /** Reconfigure a load balancer instance, if necessary */ private Optional<LoadBalancerInstance> configureInstance(LoadBalancerId id, NodeList nodes, LoadBalancer currentLoadBalancer, ZoneEndpoint zoneEndpoint, @@ -268,14 +268,11 @@ public class LoadBalancerProvisioner { id.application().serializedForm()) .value(); Set<Real> reals = shouldDeactivateRouting ? Set.of() : realsOf(nodes); - if (isUpToDate(currentLoadBalancer, reals, zoneEndpoint)) - return currentLoadBalancer.instance(); - - log.log(Level.INFO, () -> "Configuring instance for " + id + ", targeting: " + reals); + log.log(Level.FINE, () -> "Configuring instance for " + id + ", targeting: " + reals); try { - return Optional.of(service.configure(new LoadBalancerSpec(id.application(), id.cluster(), reals, zoneEndpoint, cloudAccount), - shouldDeactivateRouting || currentLoadBalancer.state() != LoadBalancer.State.active) - .withServiceIds(currentLoadBalancer.instance().map(LoadBalancerInstance::serviceIds).orElse(List.of()))); + return Optional.of(service.configure(currentLoadBalancer.instance().orElseThrow(() -> new IllegalArgumentException("expected existing instance for " + id)), + new LoadBalancerSpec(id.application(), id.cluster(), reals, zoneEndpoint, cloudAccount), + shouldDeactivateRouting || currentLoadBalancer.state() != LoadBalancer.State.active)); } catch (Exception e) { log.log(Level.WARNING, e, () -> "Could not (re)configure " + id + ", targeting: " + reals + ". The operation will be retried on next deployment"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index ab5cd577ea1..c6971f0fe02 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -95,6 +95,7 @@ class NodeAllocation { this.requiredHostFlavor = Optional.of(PermanentFlags.HOST_FLAVOR.bindTo(nodeRepository.flagSource()) .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) .with(FetchVector.Dimension.CLUSTER_TYPE, cluster.type().name()) + .with(FetchVector.Dimension.CLUSTER_ID, cluster.id().value()) .value()) .filter(s -> !s.isBlank()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java index b194730727f..fa07782057b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java @@ -181,11 +181,11 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat if ( ! lessThanHalfTheHost(this) && lessThanHalfTheHost(other)) return 1; } - // Prefer host with the least skew + // Prefer host with least skew int hostPriority = hostPriority(other); if (hostPriority != 0) return hostPriority; - // Prefer node with the cheapest flavor + // Prefer node with cheapest flavor if (this.flavor().cost() < other.flavor().cost()) return -1; if (other.flavor().cost() < this.flavor().cost()) return 1; @@ -199,7 +199,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat return Integer.compare(this.allocation().get().membership().index(), other.allocation().get().membership().index()); - // Prefer host with the latest OS version + // Prefer host with latest OS version Version thisHostOsVersion = this.parent.flatMap(host -> host.status().osVersion().current()).orElse(Version.emptyVersion); Version otherHostOsVersion = other.parent.flatMap(host -> host.status().osVersion().current()).orElse(Version.emptyVersion); if (thisHostOsVersion.isAfter(otherHostOsVersion)) return -1; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java index 1acb69113c1..4d33e1c7bad 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java @@ -88,10 +88,13 @@ public class NodeResourceLimits { return 4; } - private double minRealVcpu(ApplicationId applicationId, ClusterSpec cluster) { return minAdvertisedVcpu(applicationId, cluster); } + private double minRealVcpu(ApplicationId applicationId, ClusterSpec cluster) { + return minAdvertisedVcpu(applicationId, cluster); + } private double minRealMemoryGb(ClusterSpec cluster) { - return minAdvertisedMemoryGb(cluster) - 1.7; + if (cluster.type() == ClusterSpec.Type.admin) return 0.95; // TODO: Increase to 1.05 after March 2023 + return 2.3; } private double minRealDiskGb() { return 6; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java index d083d81c196..15a6b6ba523 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java @@ -6,10 +6,10 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.OsVersion; import com.yahoo.vespa.hosted.provision.node.Status; @@ -32,38 +32,38 @@ public class ProvisionedHost { private final NodeType hostType; private final Optional<ApplicationId> exclusiveToApplicationId; private final Optional<ClusterSpec.Type> exclusiveToClusterType; - private final List<HostName> nodeHostnames; + private final List<Address> nodeAddresses; private final NodeResources nodeResources; private final Version osVersion; private final CloudAccount cloudAccount; public ProvisionedHost(String id, String hostHostname, Flavor hostFlavor, NodeType hostType, Optional<ApplicationId> exclusiveToApplicationId, Optional<ClusterSpec.Type> exclusiveToClusterType, - List<HostName> nodeHostnames, NodeResources nodeResources, Version osVersion, CloudAccount cloudAccount) { + List<Address> nodeAddresses, NodeResources nodeResources, Version osVersion, CloudAccount cloudAccount) { this.id = Objects.requireNonNull(id, "Host id must be set"); this.hostHostname = Objects.requireNonNull(hostHostname, "Host hostname must be set"); this.hostFlavor = Objects.requireNonNull(hostFlavor, "Host flavor must be set"); this.hostType = Objects.requireNonNull(hostType, "Host type must be set"); this.exclusiveToApplicationId = Objects.requireNonNull(exclusiveToApplicationId, "exclusiveToApplicationId must be set"); this.exclusiveToClusterType = Objects.requireNonNull(exclusiveToClusterType, "exclusiveToClusterType must be set"); - this.nodeHostnames = validateNodeAddresses(nodeHostnames); + this.nodeAddresses = validateNodeAddresses(nodeAddresses); this.nodeResources = Objects.requireNonNull(nodeResources, "Node resources must be set"); this.osVersion = Objects.requireNonNull(osVersion, "OS version must be set"); this.cloudAccount = Objects.requireNonNull(cloudAccount, "Cloud account must be set"); if (!hostType.isHost()) throw new IllegalArgumentException(hostType + " is not a host"); } - private static List<HostName> validateNodeAddresses(List<HostName> nodeHostnames) { - Objects.requireNonNull(nodeHostnames, "Node hostnames must be set"); - if (nodeHostnames.isEmpty()) { - throw new IllegalArgumentException("There must be at least one node hostname"); + private static List<Address> validateNodeAddresses(List<Address> nodeAddresses) { + Objects.requireNonNull(nodeAddresses, "Node addresses must be set"); + if (nodeAddresses.isEmpty()) { + throw new IllegalArgumentException("There must be at least one node address"); } - return nodeHostnames; + return nodeAddresses; } /** Generate {@link Node} instance representing the provisioned physical host */ public Node generateHost() { - Node.Builder builder = Node.create(id, IP.Config.of(Set.of(), Set.of(), nodeHostnames), hostHostname, hostFlavor, + Node.Builder builder = Node.create(id, IP.Config.of(Set.of(), Set.of(), nodeAddresses), hostHostname, hostFlavor, hostType) .status(Status.initial().withOsVersion(OsVersion.EMPTY.withCurrent(Optional.of(osVersion)))) .cloudAccount(cloudAccount); @@ -85,12 +85,12 @@ public class ProvisionedHost { public NodeType hostType() { return hostType; } public Optional<ApplicationId> exclusiveToApplicationId() { return exclusiveToApplicationId; } public Optional<ClusterSpec.Type> exclusiveToClusterType() { return exclusiveToClusterType; } - public List<HostName> nodeHostnames() { return nodeHostnames; } + public List<Address> nodeAddresses() { return nodeAddresses; } public NodeResources nodeResources() { return nodeResources; } public Version osVersion() { return osVersion; } public CloudAccount cloudAccount() { return cloudAccount; } - public String nodeHostname() { return nodeHostnames.get(0).value(); } + public String nodeHostname() { return nodeAddresses.get(0).hostname(); } @Override public boolean equals(Object o) { @@ -103,7 +103,7 @@ public class ProvisionedHost { hostType == that.hostType && exclusiveToApplicationId.equals(that.exclusiveToApplicationId) && exclusiveToClusterType.equals(that.exclusiveToClusterType) && - nodeHostnames.equals(that.nodeHostnames) && + nodeAddresses.equals(that.nodeAddresses) && nodeResources.equals(that.nodeResources) && osVersion.equals(that.osVersion) && cloudAccount.equals(that.cloudAccount); @@ -111,7 +111,7 @@ public class ProvisionedHost { @Override public int hashCode() { - return Objects.hash(id, hostHostname, hostFlavor, hostType, exclusiveToApplicationId, exclusiveToClusterType, nodeHostnames, nodeResources, osVersion, cloudAccount); + return Objects.hash(id, hostHostname, hostFlavor, hostType, exclusiveToApplicationId, exclusiveToClusterType, nodeAddresses, nodeResources, osVersion, cloudAccount); } @Override @@ -123,7 +123,7 @@ public class ProvisionedHost { ", hostType=" + hostType + ", exclusiveToApplicationId=" + exclusiveToApplicationId + ", exclusiveToClusterType=" + exclusiveToClusterType + - ", nodeAddresses=" + nodeHostnames + + ", nodeAddresses=" + nodeAddresses + ", nodeResources=" + nodeResources + ", osVersion=" + osVersion + ", cloudAccount=" + cloudAccount + diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveResponse.java index 3cff3c5e05f..84c82d314c9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveResponse.java @@ -4,6 +4,9 @@ package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.archive.ArchiveUris; + +import java.util.Map; /** * Returns tenant archive URIs. @@ -13,11 +16,22 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; public class ArchiveResponse extends SlimeJsonResponse { public ArchiveResponse(NodeRepository nodeRepository) { + ArchiveUris archiveUris = nodeRepository.archiveUriManager().archiveUris(); Cursor archivesArray = slime.setObject().setArray("archives"); - nodeRepository.archiveUris().getArchiveUris().forEach((tenant, uri) -> { + + archiveUris.tenantArchiveUris().entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .forEach(entry -> { + Cursor archiveObject = archivesArray.addObject(); + archiveObject.setString("tenant", entry.getKey().value()); + archiveObject.setString("uri", entry.getValue()); + }); + archiveUris.accountArchiveUris().entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .forEach(entry -> { Cursor archiveObject = archivesArray.addObject(); - archiveObject.setString("tenant", tenant.value()); - archiveObject.setString("uri", uri); + archiveObject.setString("account", entry.getKey().value()); + archiveObject.setString("uri", entry.getValue()); }); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index dfe01f5f1c3..582d5963cfd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -6,7 +6,6 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.TenantName; @@ -20,6 +19,7 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeMutex; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.IP; @@ -245,15 +245,12 @@ public class NodePatcher { private Node applyIpconfigField(Node node, String name, Inspector value, LockedNodeList nodes) { switch (name) { - case "ipAddresses" -> { + case "ipAddresses": return IP.Config.verify(node.with(node.ipConfig().withPrimary(asStringSet(value))), nodes); - } - case "additionalIpAddresses" -> { + case "additionalIpAddresses": return IP.Config.verify(node.with(node.ipConfig().withPool(node.ipConfig().pool().withIpAddresses(asStringSet(value)))), nodes); - } - case "additionalHostnames" -> { - return IP.Config.verify(node.with(node.ipConfig().withPool(node.ipConfig().pool().withHostnames(asHostnames(value)))), nodes); - } + case "additionalHostnames": + return IP.Config.verify(node.with(node.ipConfig().withPool(node.ipConfig().pool().withAddresses(asAddressList(value)))), nodes); } throw new IllegalArgumentException("Could not apply field '" + name + "' on a node: No such modifiable field"); } @@ -320,19 +317,20 @@ public class NodePatcher { return strings; } - private List<HostName> asHostnames(Inspector field) { + private List<Address> asAddressList(Inspector field) { if ( ! field.type().equals(Type.ARRAY)) throw new IllegalArgumentException("Expected an ARRAY value, got a " + field.type()); - List<HostName> hostnames = new ArrayList<>(field.entries()); + List<Address> addresses = new ArrayList<>(field.entries()); for (int i = 0; i < field.entries(); i++) { Inspector entry = field.entry(i); if ( ! entry.type().equals(Type.STRING)) throw new IllegalArgumentException("Expected a STRING value, got a " + entry.type()); - hostnames.add(HostName.of(entry.asString())); + Address address = new Address(entry.asString()); + addresses.add(address); } - return hostnames; + return addresses; } private Node patchRequiredDiskSpeed(Node node, String value) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index f98c4ba1199..6bf07077c81 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -6,17 +6,18 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.serialization.NetworkPortsSerializer; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; +import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.TrustStoreItem; @@ -51,7 +52,7 @@ class NodesResponse extends SlimeJsonResponse { private final NodeFilter filter; private final boolean recursive; - private final Function<com.yahoo.vespa.applicationmodel.HostName, Optional<HostInfo>> orchestrator; + private final Function<HostName, Optional<HostInfo>> orchestrator; private final NodeRepository nodeRepository; private final StringFlag wantedDockerTagFlag; @@ -150,7 +151,7 @@ class NodesResponse extends SlimeJsonResponse { object.setString("wantedVespaVersion", allocation.membership().cluster().vespaVersion().toFullString()); NodeResourcesSerializer.toSlime(allocation.requestedResources(), object.setObject("requestedResources")); allocation.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray("networkPorts"))); - orchestrator.apply(new com.yahoo.vespa.applicationmodel.HostName(node.hostname())) + orchestrator.apply(new HostName(node.hostname())) .ifPresent(info -> { if (info.status() != HostStatus.NO_REMARKS) { object.setString("orchestratorStatus", info.status().asString()); @@ -179,12 +180,12 @@ class NodesResponse extends SlimeJsonResponse { toSlime(node.history().events(), object.setArray("history")); toSlime(node.history().log(), object.setArray("log")); ipAddressesToSlime(node.ipConfig().primary(), object.setArray("ipAddresses")); - ipAddressesToSlime(node.ipConfig().pool().asSet(), object.setArray("additionalIpAddresses")); - hostnamesToSlime(node.ipConfig().pool().hostnames(), object); + ipAddressesToSlime(node.ipConfig().pool().ipSet(), object.setArray("additionalIpAddresses")); + addressesToSlime(node.ipConfig().pool().getAddressList(), object); node.reports().toSlime(object, "reports"); node.modelName().ifPresent(modelName -> object.setString("modelName", modelName)); node.switchHostname().ifPresent(switchHostname -> object.setString("switchHostname", switchHostname)); - nodeRepository.archiveUris().archiveUriFor(node).ifPresent(uri -> object.setString("archiveUri", uri)); + nodeRepository.archiveUriManager().archiveUriFor(node).ifPresent(uri -> object.setString("archiveUri", uri)); trustedCertsToSlime(node.trustedCertificates(), object); if (!node.cloudAccount().isUnspecified()) { object.setString("cloudAccount", node.cloudAccount().value()); @@ -248,11 +249,11 @@ class NodesResponse extends SlimeJsonResponse { ipAddresses.forEach(array::addString); } - private void hostnamesToSlime(List<HostName> hostnames, Cursor object) { - if (hostnames.isEmpty()) return; + private void addressesToSlime(List<Address> addresses, Cursor object) { + if (addresses.isEmpty()) return; // When/if Address becomes richer: add another field (e.g. "addresses") and expand to array of objects Cursor addressesArray = object.setArray("additionalHostnames"); - hostnames.forEach(hostname -> addressesArray.addString(hostname.value())); + addresses.forEach(address -> addressesArray.addString(address.hostname())); } private void trustedCertsToSlime(List<TrustStoreItem> trustStoreItems, Cursor object) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index dadef5ce243..d0eb95e2d72 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; @@ -34,6 +33,7 @@ import com.yahoo.vespa.hosted.provision.NodeMutex; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.autoscale.Load; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; @@ -186,9 +186,9 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { return new MessageResponse("Updated " + patcher.application()); } } - else if (path.matches("/nodes/v2/archive/{tenant}")) { + else if (path.matches("/nodes/v2/archive/account/{key}") || path.matches("/nodes/v2/archive/tenant/{key}")) { String uri = requiredField(toSlime(request), "uri", Inspector::asString); - return setTenantArchiveUri(path.get("tenant"), Optional.of(uri)); + return setArchiveUri(path.get("key"), Optional.of(uri), !path.getPath().segments().get(3).equals("account")); } else if (path.matches("/nodes/v2/upgrade/{nodeType}")) { return setTargetVersions(path.get("nodeType"), toSlime(request)); @@ -229,7 +229,8 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { private HttpResponse handleDELETE(HttpRequest request) { Path path = new Path(request.getUri()); if (path.matches("/nodes/v2/node/{hostname}")) return deleteNode(path.get("hostname")); - if (path.matches("/nodes/v2/archive/{tenant}")) return setTenantArchiveUri(path.get("tenant"), Optional.empty()); + if (path.matches("/nodes/v2/archive/account/{key}") || path.matches("/nodes/v2/archive/tenant/{key}")) + return setArchiveUri(path.get("key"), Optional.empty(), !path.getPath().segments().get(3).equals("account")); if (path.matches("/nodes/v2/upgrade/firmware")) return cancelFirmwareCheckResponse(); throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'"); @@ -280,12 +281,12 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { Set<String> ipAddressPool = new HashSet<>(); inspector.field("additionalIpAddresses").traverse((ArrayTraverser) (i, item) -> ipAddressPool.add(item.asString())); - List<HostName> hostnames = new ArrayList<>(); + List<Address> addressPool = new ArrayList<>(); inspector.field("additionalHostnames").traverse((ArrayTraverser) (i, item) -> - hostnames.add(HostName.of(item.asString()))); + addressPool.add(new Address(item.asString()))); Node.Builder builder = Node.create(inspector.field("id").asString(), - IP.Config.of(ipAddresses, ipAddressPool, hostnames), + IP.Config.of(ipAddresses, ipAddressPool, addressPool), inspector.field("hostname").asString(), flavorFromSlime(inspector), nodeTypeFromSlime(inspector.field("type"))) @@ -422,9 +423,10 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { return new MessageResponse("Will request firmware checks on all hosts."); } - private HttpResponse setTenantArchiveUri(String tenant, Optional<String> archiveUri) { - nodeRepository.archiveUris().setArchiveUri(TenantName.from(tenant), archiveUri); - return new MessageResponse(archiveUri.map(a -> "Updated").orElse("Removed") + " archive URI for " + tenant); + private HttpResponse setArchiveUri(String key, Optional<String> archiveUri, boolean isTenant) { + if (isTenant) nodeRepository.archiveUriManager().setArchiveUri(TenantName.from(key), archiveUri); + else nodeRepository.archiveUriManager().setArchiveUri(CloudAccount.from(key), archiveUri); + return new MessageResponse(archiveUri.map(a -> "Updated").orElse("Removed") + " archive URI for " + key); } private static String hostnamesAsString(List<Node> nodes) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java index f0f85b6523f..4f88a10dff0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.testutils; import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.SystemName; /** * For running NodeRepository API with some mocked data. @@ -11,12 +12,15 @@ import com.yahoo.config.provision.CloudAccount; */ public class ContainerConfig { - public static String servicesXmlV2(int port, CloudAccount cloudAccount) { + public static String servicesXmlV2(int port, SystemName systemName, CloudAccount cloudAccount) { return """ <container version='1.0'> <config name="container.handler.threadpool"> <maxthreads>20</maxthreads> </config> + <config name="cloud.config.configserver"> + <system>%s</system> + </config> <config name="config.provisioning.cloud"> <account>%s</account> </config> @@ -47,7 +51,7 @@ public class ContainerConfig { <server id='myServer' port='%s'/> </http> </container> - """.formatted(cloudAccount.value(), port); + """.formatted(systemName.value(), cloudAccount.value(), port); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java index 84856ab310b..7e83e265496 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java @@ -7,16 +7,16 @@ import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostEvent; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; -import com.yahoo.vespa.hosted.provision.provisioning.HostIpConfig; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost; import java.time.Instant; @@ -46,7 +46,7 @@ public class MockHostProvisioner implements HostProvisioner { private int deprovisionedHosts = 0; private EnumSet<Behaviour> behaviours = EnumSet.noneOf(Behaviour.class); - private Optional<Flavor> hostFlavor = Optional.empty(); + private Map<ClusterSpec.Type, Flavor> hostFlavors = new HashMap<>(); public MockHostProvisioner(List<Flavor> flavors, MockNameResolver nameResolver, int memoryTaxGb) { this.flavors = List.copyOf(flavors); @@ -65,13 +65,16 @@ public class MockHostProvisioner implements HostProvisioner { @Override public void provisionHosts(List<Integer> provisionIndices, NodeType hostType, NodeResources resources, ApplicationId applicationId, Version osVersion, HostSharing sharing, - Optional<ClusterSpec.Type> clusterType, CloudAccount cloudAccount, - Consumer<List<ProvisionedHost>> provisionedHostsConsumer) { - Flavor hostFlavor = this.hostFlavor.orElseGet(() -> flavors.stream() - .filter(f -> sharing == HostSharing.exclusive ? compatible(f, resources) - : f.resources().satisfies(resources)) - .findFirst() - .orElseThrow(() -> new NodeAllocationException("No host flavor matches " + resources, true))); + Optional<ClusterSpec.Type> clusterType, Optional<ClusterSpec.Id> clusterId, + CloudAccount cloudAccount, Consumer<List<ProvisionedHost>> provisionedHostsConsumer) { + Flavor hostFlavor = hostFlavors.get(clusterType.orElse(ClusterSpec.Type.content)); + if (hostFlavor == null) + hostFlavor = flavors.stream() + .filter(f -> sharing == HostSharing.exclusive ? compatible(f, resources) + : f.resources().satisfies(resources)) + .findFirst() + .orElseThrow(() -> new NodeAllocationException("No host flavor matches " + resources, true)); + List<ProvisionedHost> hosts = new ArrayList<>(); for (int index : provisionIndices) { String hostHostname = hostType == NodeType.host ? "host" + index : hostType.name() + index; @@ -81,7 +84,7 @@ public class MockHostProvisioner implements HostProvisioner { hostType, sharing == HostSharing.exclusive ? Optional.of(applicationId) : Optional.empty(), Optional.empty(), - createHostnames(hostType, hostFlavor, index), + createAddressesForHost(hostType, hostFlavor, index), resources, osVersion, cloudAccount)); @@ -91,16 +94,16 @@ public class MockHostProvisioner implements HostProvisioner { } @Override - public HostIpConfig provision(Node host, Set<Node> children) throws FatalProvisioningException { + public List<Node> provision(Node host, Set<Node> children) throws FatalProvisioningException { if (behaviours.contains(Behaviour.failProvisioning)) throw new FatalProvisioningException("Failed to provision node(s)"); if (host.state() != Node.State.provisioned) throw new IllegalStateException("Host to provision must be in " + Node.State.provisioned); - Map<String, IP.Config> result = new HashMap<>(); - result.put(host.hostname(), createIpConfig(host)); + List<Node> result = new ArrayList<>(); + result.add(withIpAssigned(host)); for (var child : children) { if (child.state() != Node.State.reserved) throw new IllegalStateException("Child to provisioned must be in " + Node.State.reserved); - result.put(child.hostname(), createIpConfig(child)); + result.add(withIpAssigned(child)); } - return new HostIpConfig(result); + return result; } @Override @@ -152,14 +155,30 @@ public class MockHostProvisioner implements HostProvisioner { return this; } - public MockHostProvisioner overrideHostFlavor(String flavorName) { + public MockHostProvisioner setHostFlavor(String flavorName, ClusterSpec.Type ... types) { Flavor flavor = flavors.stream().filter(f -> f.name().equals(flavorName)) .findFirst() .orElseThrow(() -> new IllegalArgumentException("No such flavor '" + flavorName + "'")); - hostFlavor = Optional.of(flavor); + if (types.length == 0) + types = ClusterSpec.Type.values(); + for (var type : types) + hostFlavors.put(type, flavor); return this; } + /** Sets the host flavor to use to the flavor matching these resources exactly, if any. */ + public MockHostProvisioner setHostFlavorIfAvailable(NodeResources flavorAdvertisedResources, HostResourcesCalculator calculator, ClusterSpec.Type ... types) { + Optional<Flavor> hostFlavor = flavors.stream().filter(f -> calculator.advertisedResourcesOf(f).compatibleWith(flavorAdvertisedResources)) + .findFirst(); + if (types.length == 0) + types = ClusterSpec.Type.values(); + for (var type : types) + hostFlavor.ifPresent(f -> hostFlavors.put(type, f)); + return this; + } + + public Optional<Flavor> getHostFlavor(ClusterSpec.Type type) { return Optional.ofNullable(hostFlavors.get(type)); } + public MockHostProvisioner addEvent(HostEvent event) { hostEvents.add(event); return this; @@ -176,21 +195,21 @@ public class MockHostProvisioner implements HostProvisioner { return flavor.resources().compatibleWith(resourcesToVerify); } - private List<HostName> createHostnames(NodeType hostType, Flavor flavor, int hostIndex) { + private List<Address> createAddressesForHost(NodeType hostType, Flavor flavor, int hostIndex) { long numAddresses = Math.max(2, Math.round(flavor.resources().bandwidthGbps())); return IntStream.range(1, (int) numAddresses) .mapToObj(i -> { String hostname = hostType == NodeType.host ? "host" + hostIndex + "-" + i : hostType.childNodeType().name() + i; - return HostName.of(hostname); + return new Address(hostname); }) .toList(); } - public IP.Config createIpConfig(Node node) { + public Node withIpAssigned(Node node) { if (!node.type().isHost()) { - return node.ipConfig().withPrimary(nameResolver.resolveAll(node.hostname())); + return node.with(node.ipConfig().withPrimary(nameResolver.resolveAll(node.hostname()))); } int hostIndex = Integer.parseInt(node.hostname().replaceAll("^[a-z]+|-\\d+$", "")); Set<String> addresses = Set.of("::" + hostIndex + ":0"); @@ -204,7 +223,7 @@ public class MockHostProvisioner implements HostProvisioner { } } IP.Pool pool = node.ipConfig().pool().withIpAddresses(ipAddressPool); - return node.ipConfig().withPrimary(addresses).withPool(pool); + return node.with(node.ipConfig().withPrimary(addresses).withPool(pool)); } public enum Behaviour { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 66d1568262b..0a614cc9b2b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.component.Version; import com.yahoo.config.provision.ActivationContext; @@ -208,7 +209,8 @@ public class MockNodeRepository extends NodeRepository { IntRange.empty(), false, true, - Optional.empty()), + Optional.empty(), + ClusterInfo.empty()), null), app1Id, provisioner); Application app1 = applications().get(app1Id).get(); Cluster cluster1 = app1.cluster(cluster1Id.id()).get(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java index 4d0b3e75740..b964bf871c1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java @@ -74,7 +74,7 @@ public class NodeRepositoryTester { private Node addNode(String id, String hostname, String parentHostname, Flavor flavor, NodeType type) { Set<String> ips = nodeRepository.nameResolver().resolveAll(hostname); - IP.Config ipConfig = IP.Config.of(ips, type.isHost() ? ips : Set.of()); + IP.Config ipConfig = new IP.Config(ips, type.isHost() ? ips : Set.of()); Node node = Node.create(id, ipConfig, hostname, flavor, type).parentHostname(parentHostname).build(); return nodeRepository.nodes().addNodes(List.of(node), Agent.system).get(0); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java new file mode 100644 index 00000000000..894c0be2f54 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java @@ -0,0 +1,75 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.archive; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Cloud; +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Allocation; +import com.yahoo.vespa.hosted.provision.node.Generation; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; +import org.junit.Test; + +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; + +/** + * @author freva + */ +public class ArchiveUriManagerTest { + + @Test + public void archive_uri() { + ApplicationId app1 = ApplicationId.from("vespa", "music", "main"); + ApplicationId app2 = ApplicationId.from("yahoo", "music", "main"); + CloudAccount account1 = CloudAccount.from("123456789012"); + CloudAccount account2 = CloudAccount.from("210987654321"); + CloudAccount accountSystem = CloudAccount.from("555444333222"); + ArchiveUriManager archiveUriManager = new ProvisioningTester.Builder() + .zone(new Zone(Cloud.builder().account(accountSystem).build(), SystemName.Public, Environment.prod, RegionName.defaultName())) + .build().nodeRepository().archiveUriManager(); + + // Initially no uris are set + assertFalse(archiveUriManager.archiveUriFor(createNode(null, null)).isPresent()); + assertFalse(archiveUriManager.archiveUriFor(createNode(app1, account1)).isPresent()); + + archiveUriManager.setArchiveUri(app1.tenant(), Optional.of("scheme://tenant-bucket/dir")); + archiveUriManager.setArchiveUri(account1, Optional.of("scheme://account-bucket/dir")); + assertThrows(IllegalArgumentException.class, () -> archiveUriManager.setArchiveUri(accountSystem, Optional.of("scheme://something"))); + assertThrows(IllegalArgumentException.class, () -> archiveUriManager.setArchiveUri(CloudAccount.empty, Optional.of("scheme://something"))); + + assertFalse(archiveUriManager.archiveUriFor(createNode(null, null)).isPresent()); // Not allocated + assertFalse(archiveUriManager.archiveUriFor(createNode(null, account1)).isPresent()); // URI set for this account, but not allocated + assertFalse(archiveUriManager.archiveUriFor(createNode(null, account2)).isPresent()); // Not allocated + assertFalse(archiveUriManager.archiveUriFor(createNode(app2, null)).isPresent()); // No URI set for this tenant or account + assertEquals("scheme://tenant-bucket/dir/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, null)).get()); + assertEquals("scheme://account-bucket/dir/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, account1)).get()); // Account has precedence + assertFalse(archiveUriManager.archiveUriFor(createNode(app1, account2)).isPresent()); // URI set for this tenant, but is ignored because enclave account + assertEquals("scheme://tenant-bucket/dir/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, accountSystem)).get()); // URI for tenant because non-enclave acocunt + } + + private Node createNode(ApplicationId appId, CloudAccount account) { + Node.Builder nodeBuilder = Node.create("id", "h432a.prod.us-south-1.vespa.domain.tld", new Flavor(NodeResources.unspecified()), Node.State.parked, NodeType.tenant); + Optional.ofNullable(appId) + .map(app -> new Allocation(app, + ClusterMembership.from("container/default/0/0", Version.fromString("1.2.3"), Optional.empty()), + NodeResources.unspecified(), + Generation.initial(), + false)) + .ifPresent(nodeBuilder::allocation); + Optional.ofNullable(account).ifPresent(nodeBuilder::cloudAccount); + return nodeBuilder.build(); + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUrisTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUrisTest.java new file mode 100644 index 00000000000..9770d60b27a --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUrisTest.java @@ -0,0 +1,49 @@ +package com.yahoo.vespa.hosted.provision.archive; + + +import com.yahoo.config.provision.TenantName; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.provision.archive.ArchiveUris.normalizeUri; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class ArchiveUrisTest { + + @Test + void normalize_test() { + assertEquals("ftp://domain/legal-dir123/", normalizeUri("ftp://domain/legal-dir123")); + assertEquals("ftp://domain/legal-dir123/", normalizeUri("ftp://domain/legal-dir123/")); + assertEquals("s3://my-bucket-prod.region/my-tenant-123/", normalizeUri("s3://my-bucket-prod.region/my-tenant-123/")); + assertEquals("s3://my-bucket-prod.region/my-tenant_123/", normalizeUri("s3://my-bucket-prod.region/my-tenant_123/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("domain/dir/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp:/domain/dir/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp:/domain//dir/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/illegal:dir/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/-illegal-dir/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/_illegal-dir/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/illegal-dir-/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/illegal-dir_/")); + } + + @Test + void updates_in_place_if_possible() { + TenantName t1 = TenantName.from("t1"); + + ArchiveUris uris0 = new ArchiveUris(Map.of(), Map.of()); + ArchiveUris uris1 = uris0.with(t1, Optional.empty()); + assertSame(uris0, uris1); + + ArchiveUris uris2 = uris0.with(t1, Optional.of("scheme://test123")); + assertEquals(Map.of(t1, "scheme://test123/"), uris2.tenantArchiveUris()); + + assertSame(uris2, uris2.with(t1, Optional.of("scheme://test123"))); + assertSame(uris2, uris2.with(t1, Optional.of("scheme://test123/"))); + assertEquals(uris0, uris2.with(t1, Optional.empty())); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java index 158c5116e19..3ee72c18318 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.NodeResources; import com.yahoo.test.ManualClock; +import com.yahoo.vespa.hosted.provision.provisioning.DynamicProvisioningTester; import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; import org.junit.Test; @@ -22,12 +23,12 @@ public class AutoscalingIntegrationTest { @Test public void testComponentIntegration() { - var fixture = AutoscalingTester.fixture() - .hostCount(20) - .hostFlavors(new NodeResources(3, 20, 200, 1)) - .initialResources(Optional.of(new ClusterResources(2, 1, + var fixture = DynamicProvisioningTester.fixture() + .hostCount(20) + .hostFlavors(new NodeResources(3, 20, 200, 1)) + .initialResources(Optional.of(new ClusterResources(2, 1, new NodeResources(1, 10, 100, 1)))) - .build(); + .build(); MetricsV2MetricsFetcher fetcher = new MetricsV2MetricsFetcher(fixture.tester().nodeRepository(), new OrchestratorMock(), new MockHttpClient(fixture.tester().clock())); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index c19db34691a..d69d9267cfd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; @@ -12,6 +13,7 @@ import com.yahoo.config.provision.NodeResources.StorageType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.provisioning.CapacityPolicies; +import com.yahoo.vespa.hosted.provision.provisioning.DynamicProvisioningTester; import org.junit.Test; import java.time.Duration; @@ -34,12 +36,12 @@ public class AutoscalingTest { var min = new ClusterResources( 8, 1, resources); var now = new ClusterResources(12, 1, resources.with(StorageType.remote)); var max = new ClusterResources(12, 1, resources); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .clusterType(ClusterSpec.Type.content) - .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .clusterType(ClusterSpec.Type.content) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max)) + .build(); fixture.tester.clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(0.17f, 0.17, 0.12), 1, true, true, 100); var result = fixture.autoscale(); @@ -55,7 +57,7 @@ public class AutoscalingTest { @Test public void test_autoscaling_single_content_group() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); fixture.loader().applyCpuLoad(0.7f, 10); var scaledResources = fixture.tester().assertResources("Scaling up since resource usage is too high", @@ -83,20 +85,20 @@ public class AutoscalingTest { /** Using too many resources for a short period is proof we should scale up regardless of the time that takes. */ @Test public void test_no_autoscaling_with_no_measurements() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); assertTrue(fixture.autoscale().resources().isEmpty()); } @Test public void test_no_autoscaling_with_no_measurements_exclusive() { - var fixture = AutoscalingTester.fixture().awsProdSetup(false).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(false).build(); assertTrue(fixture.autoscale().resources().isEmpty()); } /** Using too many resources for a short period is proof we should scale up regardless of the time that takes. */ @Test public void test_autoscaling_up_is_fast() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); fixture.loader().applyLoad(new Load(0.1, 0.1, 0.1), 3); fixture.loader().applyLoad(new Load(1.0, 1.0, 1.0), 1); fixture.tester().assertResources("Scaling up since resource usage is too high", @@ -109,12 +111,12 @@ public class AutoscalingTest { var min = new ClusterResources(2, 1, new NodeResources(4, 8, 50, 0.1)); var now = new ClusterResources(8, 1, new NodeResources(4, 8, 50, 0.1)); var max = new ClusterResources(8, 1, new NodeResources(4, 8, 50, 0.1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(false) - .clusterType(ClusterSpec.Type.container) - .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(false) + .clusterType(ClusterSpec.Type.container) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max)) + .build(); fixture.tester().setScalingDuration(fixture.applicationId(), fixture.clusterSpec.id(), Duration.ofMinutes(5)); fixture.loader().applyLoad(new Load(0.01, 0.38, 0), 5); @@ -127,12 +129,12 @@ public class AutoscalingTest { public void initial_deployment_with_host_sharing_flag() { var min = new ClusterResources(7, 1, new NodeResources(2.0, 10.0, 384.0, 0.1)); var max = new ClusterResources(7, 1, new NodeResources(2.4, 32.0, 768.0, 0.1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(false) - .capacity(Capacity.from(min, max)) - .initialResources(Optional.empty()) - .hostSharingFlag() - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(false) + .capacity(Capacity.from(min, max)) + .initialResources(Optional.empty()) + .hostSharingFlag() + .build(); fixture.tester().assertResources("Initial resources at min, since flag turns on host sharing", 7, 1, 2.0, 10.0, 384.0, fixture.currentResources().advertisedResources()); @@ -142,13 +144,13 @@ public class AutoscalingTest { public void initial_deployment_with_host_sharing_flag_and_too_small_min() { var min = new ClusterResources(1, 1, new NodeResources(0.5, 4.0, 10, 0.1)); var max = new ClusterResources(1, 1, new NodeResources(2.0, 8.0, 50, 0.1)); - var fixture = AutoscalingTester.fixture() - .awsSetup(false, Environment.test) - .clusterType(ClusterSpec.Type.container) - .capacity(Capacity.from(min, max)) - .initialResources(Optional.empty()) - .hostSharingFlag() - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsSetup(false, Environment.test) + .clusterType(ClusterSpec.Type.container) + .capacity(Capacity.from(min, max)) + .initialResources(Optional.empty()) + .hostSharingFlag() + .build(); fixture.tester().assertResources("Initial resources at min, since flag turns on host sharing", 1, 1, 0.5, 4.0, 10.0, fixture.currentResources().advertisedResources()); @@ -157,7 +159,7 @@ public class AutoscalingTest { /** When scaling up, disregard underutilized dimensions (memory here) */ @Test public void test_only_autoscaling_up_quickly() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); fixture.loader().applyLoad(new Load(1.0, 0.1, 1.0), 10); fixture.tester().assertResources("Scaling up (only) since resource usage is too high", 8, 1, 7.1, 8.8, 75.4, @@ -167,7 +169,7 @@ public class AutoscalingTest { /** When ok to scale down, scale in both directions simultaneously (compare to test_only_autoscaling_up_quickly) */ @Test public void test_scale_in_both_directions_when_ok_to_scale_down() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); fixture.tester.clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(1.0, 0.1, 1.0), 10); fixture.tester().assertResources("Scaling cpu and disk up and memory down", @@ -177,7 +179,7 @@ public class AutoscalingTest { @Test public void test_scale_in_both_directions_when_ok_to_scale_down_exclusive() { - var fixture = AutoscalingTester.fixture().awsProdSetup(false).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(false).build(); fixture.tester.clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(1.0, 0.1, 1.0), 10); fixture.tester().assertResources("Scaling cpu and disk up, memory follows", @@ -187,7 +189,7 @@ public class AutoscalingTest { @Test public void test_autoscaling_uses_peak() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); fixture.loader().applyCpuLoad(0.01, 100); fixture.loader().applyCpuLoad(0.70, 1); fixture.loader().applyCpuLoad(0.01, 100); @@ -198,7 +200,7 @@ public class AutoscalingTest { @Test public void test_autoscaling_uses_peak_exclusive() { - var fixture = AutoscalingTester.fixture().awsProdSetup(false).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(false).build(); fixture.loader().applyCpuLoad(0.01, 100); fixture.loader().applyCpuLoad(0.70, 1); fixture.loader().applyCpuLoad(0.01, 100); @@ -209,7 +211,7 @@ public class AutoscalingTest { @Test public void test_autoscaling_uses_peak_preprovisioned() { - var fixture = AutoscalingTester.fixture().hostCount(15).build(); + var fixture = DynamicProvisioningTester.fixture().hostCount(15).build(); fixture.loader().applyCpuLoad(0.01, 100); fixture.loader().applyCpuLoad(0.70, 1); fixture.loader().applyCpuLoad(0.01, 100); @@ -223,10 +225,10 @@ public class AutoscalingTest { var min = new ClusterResources(1, 1, new NodeResources(0.5, 4, 10, 0.3)); var now = new ClusterResources(4, 1, new NodeResources(8, 16, 10, 0.3)); var max = new ClusterResources(4, 1, new NodeResources(16, 32, 50, 0.3)); - var fixture = AutoscalingTester.fixture(min, now, max) - .clusterType(ClusterSpec.Type.container) - .awsProdSetup(false) - .build(); + var fixture = DynamicProvisioningTester.fixture(min, now, max) + .clusterType(ClusterSpec.Type.container) + .awsProdSetup(false) + .build(); var duration = fixture.loader().addMeasurements(new Load(0.04, 0.39, 0.01), 20); fixture.tester().clock().advance(duration.negated()); fixture.loader().zeroTraffic(20, 1); @@ -238,7 +240,7 @@ public class AutoscalingTest { /** We prefer fewer nodes for container clusters as (we assume) they all use the same disk and memory */ @Test public void test_autoscaling_single_container_group() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).clusterType(ClusterSpec.Type.container).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).clusterType(ClusterSpec.Type.container).build(); fixture.loader().applyCpuLoad(0.25f, 120); ClusterResources scaledResources = fixture.tester().assertResources("Scaling cpu up", @@ -255,12 +257,12 @@ public class AutoscalingTest { @Test public void autoscaling_handles_disk_setting_changes_exclusive_preprovisioned() { var resources = new NodeResources(3, 100, 100, 1, slow); - var fixture = AutoscalingTester.fixture() - .hostCount(20) - .hostFlavors(resources) - .initialResources(Optional.of(new ClusterResources(5, 1, resources))) - .capacity(Capacity.from(new ClusterResources(5, 1, resources))) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .hostCount(20) + .hostFlavors(resources) + .initialResources(Optional.of(new ClusterResources(5, 1, resources))) + .capacity(Capacity.from(new ClusterResources(5, 1, resources))) + .build(); assertTrue(fixture.tester().nodeRepository().nodes().list().owner(fixture.applicationId).stream() .allMatch(n -> n.allocation().get().requestedResources().diskSpeed() == slow)); @@ -289,11 +291,11 @@ public class AutoscalingTest { NodeResources resources = new NodeResources(1, 100, 100, 1); var capacity = Capacity.from(new ClusterResources( 2, 1, resources.with(DiskSpeed.any)), new ClusterResources( 10, 1, resources.with(DiskSpeed.any))); - var fixture = AutoscalingTester.fixture() - .capacity(capacity) - .awsProdSetup(true) - .initialResources(Optional.empty()) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .capacity(capacity) + .awsProdSetup(true) + .initialResources(Optional.empty()) + .build(); // Redeployment without target: Uses current resource numbers with *requested* non-numbers (i.e disk-speed any) assertTrue(fixture.tester().nodeRepository().applications().get(fixture.applicationId).get().cluster(fixture.clusterSpec.id()).get().target().resources().isEmpty()); @@ -313,10 +315,10 @@ public class AutoscalingTest { var min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); var now = new ClusterResources(5, 1, new NodeResources(1.9, 70, 70, 1)); var max = new ClusterResources( 6, 1, new NodeResources(2.4, 78, 79, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max)).build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max)).build(); fixture.tester().clock().advance(Duration.ofDays(1)); fixture.loader().applyLoad(new Load(0.25, 0.95, 0.95), 120); @@ -329,7 +331,7 @@ public class AutoscalingTest { public void autoscaling_respects_lower_limit() { var min = new ClusterResources( 4, 1, new NodeResources(1.8, 7.4, 8.5, 1)); var max = new ClusterResources( 6, 1, new NodeResources(2.4, 78, 79, 1)); - var fixture = AutoscalingTester.fixture().awsProdSetup(true).capacity(Capacity.from(min, max)).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).capacity(Capacity.from(min, max)).build(); // deploy fixture.tester().clock().advance(Duration.ofDays(2)); @@ -343,11 +345,11 @@ public class AutoscalingTest { public void autoscaling_with_unspecified_resources_use_defaults_exclusive() { var min = new ClusterResources( 2, 1, NodeResources.unspecified()); var max = new ClusterResources( 6, 1, NodeResources.unspecified()); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(false) - .initialResources(Optional.empty()) - .capacity(Capacity.from(min, max)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(false) + .initialResources(Optional.empty()) + .capacity(Capacity.from(min, max)) + .build(); NodeResources defaultResources = new CapacityPolicies(fixture.tester().nodeRepository()).defaultNodeResources(fixture.clusterSpec, fixture.applicationId); @@ -368,11 +370,11 @@ public class AutoscalingTest { var min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1)); var now = new ClusterResources(5, 5, new NodeResources(3.0, 10, 10, 1)); var max = new ClusterResources(18, 6, new NodeResources(100, 1000, 1000, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max)) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(0.4, 240); fixture.tester().assertResources("Scaling cpu up", @@ -385,11 +387,11 @@ public class AutoscalingTest { var min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1)); var now = new ClusterResources(5, 5, new NodeResources(3.0, 10, 10, 1)); var max = new ClusterResources(18, 6, new NodeResources(100, 1000, 1000, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max, IntRange.of(2, 3), false, true, Optional.empty())) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max, IntRange.of(2, 3), false, true, Optional.empty(), ClusterInfo.empty())) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(0.4, 240); fixture.tester().assertResources("Scaling cpu up", @@ -400,7 +402,7 @@ public class AutoscalingTest { @Test public void test_autoscaling_limits_when_min_equals_max() { ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); - var fixture = AutoscalingTester.fixture().awsProdSetup(true).capacity(Capacity.from(min, min)).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).capacity(Capacity.from(min, min)).build(); fixture.tester().clock().advance(Duration.ofDays(1)); fixture.loader().applyCpuLoad(0.25, 120); @@ -412,14 +414,14 @@ public class AutoscalingTest { var resources = new ClusterResources( 2, 1, new NodeResources(3, 100, 50, 1)); var local = new NodeResources(3, 100, 75, 1, fast, StorageType.local); var remote = new NodeResources(3, 100, 50, 1, fast, StorageType.remote); - var fixture = AutoscalingTester.fixture() - .dynamicProvisioning(true) - .allowHostSharing(false) - .clusterType(ClusterSpec.Type.container) - .hostFlavors(local, remote) - .capacity(Capacity.from(resources)) - .initialResources(Optional.of(new ClusterResources(3, 1, resources.nodeResources()))) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .dynamicProvisioning(true) + .allowHostSharing(false) + .clusterType(ClusterSpec.Type.container) + .hostFlavors(local, remote) + .capacity(Capacity.from(resources)) + .initialResources(Optional.of(new ClusterResources(3, 1, resources.nodeResources()))) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(0.01, 0.01, 0.01), 120); @@ -434,7 +436,7 @@ public class AutoscalingTest { @Test public void suggestions_ignores_limits() { ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); - var fixture = AutoscalingTester.fixture().awsProdSetup(true).capacity(Capacity.from(min, min)).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).capacity(Capacity.from(min, min)).build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(1.0, 120); fixture.tester().assertResources("Suggesting above capacity limit", @@ -445,7 +447,7 @@ public class AutoscalingTest { @Test public void suggestions_ignores_limits_exclusive() { ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); - var fixture = AutoscalingTester.fixture().awsProdSetup(false).capacity(Capacity.from(min, min)).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(false).capacity(Capacity.from(min, min)).build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(1.0, 120); fixture.tester().assertResources("Suggesting above capacity limit", @@ -455,7 +457,7 @@ public class AutoscalingTest { @Test public void not_using_out_of_service_measurements() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(0.9, 0.6, 0.7), 1, false, true, 120); assertTrue("Not scaling up since nodes were measured while cluster was out of service", @@ -464,7 +466,7 @@ public class AutoscalingTest { @Test public void not_using_unstable_measurements() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(0.9, 0.6, 0.7), 1, true, false, 120); assertTrue("Not scaling up since nodes were measured while cluster was unstable", @@ -476,11 +478,11 @@ public class AutoscalingTest { var min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1)); var now = new ClusterResources(5, 5, new NodeResources(3, 100, 100, 1)); var max = new ClusterResources(20, 20, new NodeResources(10, 1000, 1000, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max)) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(0.9, 120); fixture.tester().assertResources("Scaling up to 2 nodes, scaling memory and disk down at the same time", @@ -493,11 +495,11 @@ public class AutoscalingTest { var min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1)); var now = new ClusterResources(5, 5, new NodeResources(3, 100, 100, 1)); var max = new ClusterResources(20, 20, new NodeResources(10, 1000, 1000, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max, IntRange.of(1), false, true, Optional.empty())) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max, IntRange.of(1), false, true, Optional.empty(), ClusterInfo.empty())) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(0.9, 120); fixture.tester().assertResources("Scaling up to 2 nodes, scaling memory and disk down at the same time", @@ -510,11 +512,11 @@ public class AutoscalingTest { var min = new ClusterResources( 3, 1, new NodeResources(1, 1, 1, 1)); var now = new ClusterResources(6, 2, new NodeResources(3, 100, 100, 1)); var max = new ClusterResources(21, 7, new NodeResources(100, 1000, 1000, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max)) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); Duration timePassed = fixture.loader().addCpuMeasurements(0.25, 120); fixture.tester().clock().advance(timePassed.negated()); @@ -530,11 +532,11 @@ public class AutoscalingTest { var min = new ClusterResources( 3, 1, new NodeResources(1, 1, 1, 1)); var now = new ClusterResources(6, 2, new NodeResources(3, 100, 100, 1)); var max = new ClusterResources(21, 7, new NodeResources(100, 1000, 1000, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max)) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); Duration timePassed = fixture.loader().addCpuMeasurements(0.25, 120); fixture.tester().clock().advance(timePassed.negated()); @@ -549,11 +551,11 @@ public class AutoscalingTest { var min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1)); var now = new ClusterResources(6, 2, new NodeResources(10, 100, 100, 1)); var max = new ClusterResources(30, 30, new NodeResources(100, 100, 1000, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max)) + .build(); fixture.tester().clock().advance(Duration.ofDays(1)); fixture.loader().applyMemLoad(1.0, 1000); fixture.tester().assertResources("Increase group size to reduce memory load", @@ -566,11 +568,11 @@ public class AutoscalingTest { var min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); var now = new ClusterResources(6, 1, new NodeResources(3, 100, 100, 1)); var max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max)) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(0.16, 0.02, 0.5), 120); fixture.tester().assertResources("Scaling down memory", @@ -580,7 +582,7 @@ public class AutoscalingTest { @Test public void scaling_down_only_after_delay() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); fixture.loader().applyCpuLoad(0.02, 120); assertTrue("Too soon after initial deployment", fixture.autoscale().resources().isEmpty()); fixture.tester().clock().advance(Duration.ofDays(2)); @@ -594,10 +596,10 @@ public class AutoscalingTest { public void test_autoscaling_considers_read_share() { var min = new ClusterResources( 1, 1, new NodeResources(3, 100, 100, 1)); var max = new ClusterResources(10, 1, new NodeResources(3, 100, 100, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .capacity(Capacity.from(min, max)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .capacity(Capacity.from(min, max)) + .build(); fixture.tester.clock().advance(Duration.ofDays(1)); fixture.loader().applyCpuLoad(0.25, 120); @@ -622,7 +624,7 @@ public class AutoscalingTest { @Test public void test_autoscaling_considers_growth_rate() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); fixture.tester().clock().advance(Duration.ofDays(2)); Duration timeAdded = fixture.loader().addLoadMeasurements(100, t -> t == 0 ? 200.0 : 100.0, t -> 0.0); @@ -656,7 +658,7 @@ public class AutoscalingTest { @Test public void test_autoscaling_weights_growth_rate_by_confidence() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); double scalingFactor = 1.0/6000; // To make the average query rate low fixture.setScalingDuration(Duration.ofMinutes(60)); @@ -673,7 +675,7 @@ public class AutoscalingTest { @Test public void test_autoscaling_considers_query_vs_write_rate() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); fixture.loader().addCpuMeasurements(0.4, 220); @@ -724,10 +726,10 @@ public class AutoscalingTest { @Test public void test_autoscaling_in_dev_preprovisioned() { - var fixture = AutoscalingTester.fixture() - .hostCount(5) - .zone(new Zone(Environment.dev, RegionName.from("us-east"))) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .hostCount(5) + .zone(new Zone(Environment.dev, RegionName.from("us-east"))) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(1.0, 1.0, 1.0), 200); assertTrue("Not attempting to scale up because policies dictate we'll only get one node", @@ -740,10 +742,10 @@ public class AutoscalingTest { new NodeResources(1, 4, 10, 1, NodeResources.DiskSpeed.any)); var max = new ClusterResources(20, 20, new NodeResources(100, 1000, 1000, 1, NodeResources.DiskSpeed.any)); - var fixture = AutoscalingTester.fixture() - .awsSetup(true, Environment.dev) - .capacity(Capacity.from(min, max, IntRange.of(3, 5), false, true, Optional.empty())) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsSetup(true, Environment.dev) + .capacity(Capacity.from(min, max, IntRange.of(3, 5), false, true, Optional.empty(), ClusterInfo.empty())) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(1.0, 1.0, 1.0), 200); fixture.tester().assertResources("Scale only to a single node and group since this is dev", @@ -762,13 +764,14 @@ public class AutoscalingTest { IntRange.empty(), true, true, - Optional.empty()); - - var fixture = AutoscalingTester.fixture() - .hostCount(5) - .capacity(requiredCapacity) - .zone(new Zone(Environment.dev, RegionName.from("us-east"))) - .build(); + Optional.empty(), + ClusterInfo.empty()); + + var fixture = DynamicProvisioningTester.fixture() + .hostCount(5) + .capacity(requiredCapacity) + .zone(new Zone(Environment.dev, RegionName.from("us-east"))) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(1.0, 1.0, 1.0), 200); fixture.tester().assertResources("We scale even in dev because resources are 'required'", @@ -784,13 +787,14 @@ public class AutoscalingTest { IntRange.empty(), true, true, - Optional.empty()); - - var fixture = AutoscalingTester.fixture() - .hostCount(5) - .capacity(requiredCapacity) - .zone(new Zone(Environment.dev, RegionName.from("us-east"))) - .build(); + Optional.empty(), + ClusterInfo.empty()); + + var fixture = DynamicProvisioningTester.fixture() + .hostCount(5) + .capacity(requiredCapacity) + .zone(new Zone(Environment.dev, RegionName.from("us-east"))) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(1.0, 1.0, 1.0), 200); fixture.tester().assertResources("We scale even in dev because resources are required", @@ -802,12 +806,12 @@ public class AutoscalingTest { public void test_changing_exclusivity() { var min = new ClusterResources( 2, 1, new NodeResources( 3, 4, 100, 1)); var max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .cluster(clusterSpec(true)) - .capacity(Capacity.from(min, max)) - .initialResources(Optional.empty()) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .cluster(clusterSpec(true)) + .capacity(Capacity.from(min, max)) + .initialResources(Optional.empty()) + .build(); fixture.tester().assertResources("Initial deployment at minimum", 2, 1, 4, 8, 100, fixture.currentResources().advertisedResources()); @@ -831,11 +835,11 @@ public class AutoscalingTest { var min = new ClusterResources(7, 1, new NodeResources( 2, 10, 384, 1)); var now = new ClusterResources(7, 1, new NodeResources( 3.4, 16.2, 450.1, 1)); var max = new ClusterResources(7, 1, new NodeResources( 4, 32, 768, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .capacity(Capacity.from(min, max)) - .initialResources(Optional.of(now)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .capacity(Capacity.from(min, max)) + .initialResources(Optional.of(now)) + .build(); var initialNodes = fixture.nodes().asList(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(0.06, 0.52, 0.27), 100); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java index eb2488b7829..704491ed44f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java @@ -1,15 +1,20 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.applications.BcpGroupInfo; +import com.yahoo.vespa.hosted.provision.provisioning.DynamicProvisioningTester; import org.junit.Test; import java.time.Duration; import java.util.Optional; +import static org.junit.Assert.assertEquals; + /** * Tests autoscaling using information from the BCP group this cluster deployment * is part of to supplement local data when the local deployment lacks sufficient traffic. @@ -21,13 +26,13 @@ public class AutoscalingUsingBcpGroupInfoTest { /** Tests with varying BCP group info parameters. */ @Test public void test_autoscaling_single_content_group() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.store(new BcpGroupInfo(100, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 3.6, 6.1, 25.3, + 9, 1, 3.6, 6.1, 25.3, fixture.autoscale()); // Higher query rate @@ -35,7 +40,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(200, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 7.1, 6.1, 25.3, + 9, 1, 7.1, 6.1, 25.3, fixture.autoscale()); // Higher headroom @@ -43,7 +48,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.3, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 4.2, 6.1, 25.3, + 9, 1, 4.2, 6.1, 25.3, fixture.autoscale()); // Higher per query cost @@ -51,7 +56,15 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.45)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 5.4, 6.1, 25.3, + 9, 1, 5.4, 6.1, 25.3, + fixture.autoscale()); + + // Bcp elsewhere is 0 - use local only + fixture.tester().clock().advance(Duration.ofDays(2)); + fixture.store(new BcpGroupInfo(0, 1.1, 0.45)); + fixture.loader().addCpuMeasurements(0.7f, 10); + fixture.tester().assertResources("Scaling using local info", + 8, 1, 1, 7.0, 29.0, fixture.autoscale()); } @@ -62,17 +75,17 @@ public class AutoscalingUsingBcpGroupInfoTest { new NodeResources(1, 4, 10, 1, NodeResources.DiskSpeed.any)); var max = new ClusterResources(21, 3, new NodeResources(100, 1000, 1000, 1, NodeResources.DiskSpeed.any)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .initialResources(Optional.of(new ClusterResources(9, 3, new NodeResources(2, 16, 75, 1)))) - .capacity(Capacity.from(min, max)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .initialResources(Optional.of(new ClusterResources(9, 3, new NodeResources(2, 16, 75, 1)))) + .capacity(Capacity.from(min, max)) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.store(new BcpGroupInfo(100, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 3, 3, 10.5, 41.0, 168.9, + 3, 3, 10.5, 41.0, 168.9, fixture.autoscale()); // Higher query rate @@ -80,7 +93,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(200, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 3, 3, 20.9, 41.0, 168.9, + 3, 3, 20.9, 41.0, 168.9, fixture.autoscale()); // Higher headroom @@ -88,7 +101,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.3, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 3, 3, 12.4, 41.0, 168.9, + 3, 3, 12.4, 41.0, 168.9, fixture.autoscale()); // Higher per query cost @@ -96,7 +109,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.45)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 3, 3, 15.7, 41.0, 168.9, + 3, 3, 15.7, 41.0, 168.9, fixture.autoscale()); } @@ -108,13 +121,13 @@ public class AutoscalingUsingBcpGroupInfoTest { */ @Test public void test_autoscaling_container() { - var fixture = AutoscalingTester.fixture().clusterType(ClusterSpec.Type.container).awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().clusterType(ClusterSpec.Type.container).awsProdSetup(true).build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.store(new BcpGroupInfo(100, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 4.0, 16.0, 40.8, + 8, 1, 4.0, 16.0, 40.8, fixture.autoscale()); // Higher query rate (mem and disk changes are due to being assigned larger hosts where we get less overhead share @@ -122,7 +135,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(200, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 8.0, 16.0, 40.8, + 8, 1, 8.0, 16.0, 40.8, fixture.autoscale()); // Higher headroom @@ -130,7 +143,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.3, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 5, 1, 8.0, 16.0, 40.8, + 5, 1, 8.0, 16.0, 40.8, fixture.autoscale()); // Higher per query cost @@ -138,20 +151,42 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.45)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 6, 1, 8.0, 16.0, 40.8, + 6, 1, 8.0, 16.0, 40.8, + fixture.autoscale()); + } + + @Test + public void test_autoscaling_with_bcp_deadline() { + var capacity = Capacity.from(new ClusterResources(2, 1, + new NodeResources(1, 4, 10, 1, NodeResources.DiskSpeed.any)), + new ClusterResources(20, 1, + new NodeResources(100, 1000, 1000, 1, NodeResources.DiskSpeed.any)), + IntRange.empty(), false, true, Optional.empty(), + new ClusterInfo.Builder().bcpDeadline(Duration.ofMinutes(60)).build()); + + var fixture = DynamicProvisioningTester.fixture() + .capacity(capacity) + .clusterType(ClusterSpec.Type.container).awsProdSetup(true).build(); + + // We can rescale within deadline - do not take BCP info into account + fixture.tester().clock().advance(Duration.ofDays(2)); + fixture.store(new BcpGroupInfo(100, 1.1, 0.45)); + fixture.loader().addCpuMeasurements(0.7f, 10); + fixture.tester().assertResources("No need for traffic shift headroom", + 2, 1, 2.0, 16.0, 40.8, fixture.autoscale()); } @Test public void test_autoscaling_single_content_group_with_some_local_traffic() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); // Baseline: No local traffic, group traffic indicates much higher cpu usage than local fixture.tester().clock().advance(Duration.ofDays(2)); fixture.store(new BcpGroupInfo(200, 1.3, 0.45)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 14.2, 7.0, 29.0, + 8, 1, 14.2, 7.0, 29.0, fixture.autoscale()); // Some local traffic @@ -161,7 +196,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration1.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 10.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 6.9, 7.0, 29.0, + 8, 1, 6.9, 7.0, 29.0, fixture.autoscale()); // Enough local traffic to get half the votes @@ -171,7 +206,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration2.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 50.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 2.7, 6.1, 25.3, + 9, 1, 2.7, 6.1, 25.3, fixture.autoscale()); // Mostly local @@ -181,7 +216,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration3.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 90.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 2.1, 6.1, 25.3, + 9, 1, 2.1, 6.1, 25.3, fixture.autoscale()); // Local only @@ -191,7 +226,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration4.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 100.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 2.0, 6.1, 25.3, + 9, 1, 2.0, 6.1, 25.3, fixture.autoscale()); // No group info, should be the same as the above @@ -201,7 +236,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration5.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 100.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 2.0, 6.1, 25.3, + 9, 1, 2.0, 6.1, 25.3, fixture.autoscale()); // 40 query rate, no group info (for reference to the below) @@ -211,28 +246,65 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration6.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 40.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 1.4, 6.1, 25.3, + 9, 1, 1.4, 6.1, 25.3, fixture.autoscale()); // Local query rate is too low but global is even lower so disregard it, giving the same as above fixture.tester().clock().advance(Duration.ofDays(2)); - fixture.store(new BcpGroupInfo(200/40.0, 1.3, 0.45*40.0)); + fixture.store(new BcpGroupInfo(200 / 40.0, 1.3, 0.45 * 40.0)); Duration duration7 = fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().clock().advance(duration7.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 40.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 1.4, 6.1, 25.3, + 9, 1, 1.4, 6.1, 25.3, fixture.autoscale()); // Local query rate is too low to be fully confident, and so is global but as it is slightly larger, incorporate it slightly fixture.tester().clock().advance(Duration.ofDays(2)); - fixture.store(new BcpGroupInfo(200/4.0, 1.3, 0.45*4.0)); + fixture.store(new BcpGroupInfo(200 / 4.0, 1.3, 0.45 * 4.0)); Duration duration8 = fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().clock().advance(duration8.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 40.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 1.8, 6.1, 25.3, + 9, 1, 1.8, 6.1, 25.3, fixture.autoscale()); } + /** Tests with varying BCP group info parameters. */ + @Test + public void test_autoscaling_metrics() { + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); + + // Empty has metrics at zero + assertEquals(new Autoscaling.Metrics(0, 0, 0), + fixture.autoscale().metrics()); + + + // No external load mesurements -> 0 + fixture.tester().clock().advance(Duration.ofDays(2)); + fixture.loader().addCpuMeasurements(0.7f, 10); + assertEquals(new Autoscaling.Metrics(0, 1.0, 0), + fixture.autoscale().metrics()); + + // External load is measured to zero -> 0 + fixture.tester().clock().advance(Duration.ofDays(2)); + fixture.loader().addCpuMeasurements(0.7f, 10); + fixture.loader().addQueryRateMeasurements(10, i -> 0.0); + assertEquals(new Autoscaling.Metrics(0, 1.0, 0), + fixture.autoscale().metrics()); + + // External load + fixture.tester().clock().advance(Duration.ofDays(2)); + fixture.loader().addCpuMeasurements(0.7f, 10); + fixture.loader().addQueryRateMeasurements(10, i -> 110.0); + assertEquals(new Autoscaling.Metrics(110, 1.1, 0.05), + round(fixture.autoscale().metrics())); + } + + private Autoscaling.Metrics round(Autoscaling.Metrics metrics) { + return new Autoscaling.Metrics(Math.round(metrics.queryRate() * 100) / 100.0, + Math.round(metrics.growthRateHeadroom() * 100) / 100.0, + Math.round(metrics.cpuCostPerQuery() * 100) / 100.0); + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java index 5caf50a4e83..48db3fade95 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.Cloud; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; @@ -24,6 +25,7 @@ import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.applications.BcpGroupInfo; import com.yahoo.vespa.hosted.provision.autoscale.awsnodes.AwsHostResourcesCalculatorImpl; import com.yahoo.vespa.hosted.provision.autoscale.awsnodes.AwsNodeTypes; +import com.yahoo.vespa.hosted.provision.provisioning.DynamicProvisioningTester; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import java.time.Duration; import java.util.Arrays; @@ -37,7 +39,7 @@ import java.util.Optional; */ public class Fixture { - final AutoscalingTester tester; + final DynamicProvisioningTester tester; final Zone zone; final ApplicationId applicationId; final ClusterSpec clusterSpec; @@ -49,13 +51,13 @@ public class Fixture { applicationId = builder.application; clusterSpec = builder.cluster; capacity = builder.capacity; - tester = new AutoscalingTester(builder.zone, builder.resourceCalculator, builder.hostFlavors, builder.flagSource, hostCount); + tester = new DynamicProvisioningTester(builder.zone, builder.resourceCalculator, builder.hostFlavors, builder.flagSource, hostCount); var deployCapacity = initialResources.isPresent() ? Capacity.from(initialResources.get()) : capacity; tester.deploy(builder.application, builder.cluster, deployCapacity); this.loader = new Loader(this); } - public AutoscalingTester tester() { return tester; } + public DynamicProvisioningTester tester() { return tester; } public ApplicationId applicationId() { return applicationId; } @@ -141,7 +143,7 @@ public class Fixture { public static class Builder { - ApplicationId application = AutoscalingTester.applicationId("application1"); + ApplicationId application = DynamicProvisioningTester.applicationId("application1"); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("cluster1")).vespaVersion("7").build(); Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); List<Flavor> hostFlavors = List.of(new Flavor(new NodeResources(100, 100, 100, 1))); @@ -150,7 +152,7 @@ public class Fixture { new NodeResources(1, 4, 10, 1, NodeResources.DiskSpeed.any)), new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1, NodeResources.DiskSpeed.any))); - HostResourcesCalculator resourceCalculator = new AutoscalingTester.MockHostResourcesCalculator(zone); + HostResourcesCalculator resourceCalculator = new DynamicProvisioningTester.MockHostResourcesCalculator(zone); final InMemoryFlagSource flagSource = new InMemoryFlagSource(); int hostCount = 0; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java index e83880404f4..24697d02681 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java @@ -76,6 +76,7 @@ public class MetricsV2MetricsFetcherTest { assertEquals(0.15, values.get(0).getSecond().load().memory(), delta); assertEquals(0.20, values.get(0).getSecond().load().disk(), delta); assertEquals(3, values.get(0).getSecond().generation(), delta); + assertFalse(values.get(0).getSecond().inService()); assertTrue(values.get(0).getSecond().stable()); } @@ -108,114 +109,119 @@ public class MetricsV2MetricsFetcherTest { } final String cannedResponseForApplication1 = - "{\n" + - " \"nodes\": [\n" + - " {\n" + - " \"hostname\": \"host-1.yahoo.com\",\n" + - " \"role\": \"role0\",\n" + - " \"node\": {\n" + - " \"timestamp\": 1234,\n" + - " \"metrics\": [\n" + - " {\n" + - " \"values\": {\n" + - " \"cpu.util\": 16.2,\n" + - " \"mem.util\": 23.1,\n" + - " \"disk.util\": 82\n" + - " },\n" + - " \"dimensions\": {\n" + - " \"state\": \"active\"\n" + - " }\n" + - " }\n" + - " ]\n" + - " }\n" + - " },\n" + - " {\n" + - " \"hostname\": \"host-2.yahoo.com\",\n" + - " \"role\": \"role1\",\n" + - " \"node\": {\n" + - " \"timestamp\": 1200,\n" + - " \"metrics\": [\n" + - " {\n" + - " \"values\": {\n" + - " \"mem.util\": 30,\n" + - " \"disk.util\": 40\n" + - " },\n" + - " \"dimensions\": {\n" + - " \"state\": \"active\"\n" + - " }\n" + - " }\n" + - " ]\n" + - " },\n" + - " \"services\": [\n" + - " {\n" + - " \"name\": \"searchnode\",\n" + - " \"timestamp\": 1234,\n" + - " \"status\": {\n" + - " \"code\": \"up\"\n" + - " },\n" + - " \"metrics\": [\n" + - " {\n" + - " \"values\": {\n" + - " \"content.proton.documentdb.matching.queries.rate\": 20.5\n" + - " },\n" + - " \"dimensions\": {\n" + - " \"documentType\": \"music\"\n" + - " }\n" + - " },\n" + - " {\n" + - " \"values\": {\n" + - " \"content.proton.resource_usage.memory.average\": 0.35,\n" + - " \"content.proton.resource_usage.disk.average\": 0.45\n" + - " },\n" + - " \"dimensions\": {\n" + - " }\n" + - " },\n" + - " {\n" + - " \"values\": {\n" + - " \"content.proton.documentdb.matching.queries.rate\": 13.5\n" + - " },\n" + - " \"dimensions\": {\n" + - " \"documentType\": \"books\"\n" + - " }\n" + - " },\n" + - " {\n" + - " \"values\": {\n" + - " \"queries.rate\": 11.0\n" + - " },\n" + - " \"dimensions\": {\n" + - " }\n" + - " }\n" + - " ]\n" + - " }\n" + - " ]\n" + - " }\n" + - " ]\n" + - "}\n"; + """ + { + "nodes": [ + { + "hostname": "host-1.yahoo.com", + "role": "role0", + "node": { + "timestamp": 1234, + "metrics": [ + { + "values": { + "cpu.util": 16.2, + "mem.util": 23.1, + "disk.util": 82 + }, + "dimensions": { + "state": "active" + } + } + ] + } + }, + { + "hostname": "host-2.yahoo.com", + "role": "role1", + "node": { + "timestamp": 1200, + "metrics": [ + { + "values": { + "mem.util": 30, + "disk.util": 40 + }, + "dimensions": { + "state": "active" + } + } + ] + }, + "services": [ + { + "name": "searchnode", + "timestamp": 1234, + "status": { + "code": "up" + }, + "metrics": [ + { + "values": { + "content.proton.documentdb.matching.queries.rate": 20.5 + }, + "dimensions": { + "documentType": "music" + } + }, + { + "values": { + "content.proton.resource_usage.memory.average": 0.35, + "content.proton.resource_usage.disk.average": 0.45 + }, + "dimensions": { + } + }, + { + "values": { + "content.proton.documentdb.matching.queries.rate": 13.5 + }, + "dimensions": { + "documentType": "books" + } + }, + { + "values": { + "queries.rate": 11.0 + }, + "dimensions": { + } + } + ] + } + ] + } + ] + } + """; final String cannedResponseForApplication2 = - "{\n" + - " \"nodes\": [\n" + - " {\n" + - " \"hostname\": \"host-3.yahoo.com\",\n" + - " \"role\": \"role0\",\n" + - " \"node\": {\n" + - " \"timestamp\": 1300,\n" + - " \"metrics\": [\n" + - " {\n" + - " \"values\": {\n" + - " \"cpu.util\": 10,\n" + - " \"mem.util\": 15,\n" + - " \"disk.util\": 20,\n" + - " \"application_generation\": 3\n" + - " },\n" + - " \"dimensions\": {\n" + - " \"state\": \"active\"\n" + - " }\n" + - " }\n" + - " ]\n" + - " }\n" + - " }\n" + - " ]\n" + - "}\n"; + """ + { + "nodes": [ + { + "hostname": "host-3.yahoo.com", + "role": "role0", + "node": { + "timestamp": 1300, + "metrics": [ + { + "values": { + "cpu.util": 10, + "mem.util": 15, + "disk.util": 20, + "application_generation.last": 3, + "in_service.last": 0 + }, + "dimensions": { + "state": "active" + } + } + ] + } + } + ] + } + """; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsResourcesCalculator.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsResourcesCalculator.java index 96fa143dc57..69469bb03c7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsResourcesCalculator.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsResourcesCalculator.java @@ -48,6 +48,7 @@ public class AwsResourcesCalculator { ( hostFlavor.advertisedResources().memoryGb() - ( real ? hostMemoryOverhead : 0)); if (memoryShare > 1) // The real resources of the host cannot fit the requested real resources after overhead memoryShare = 1; + return hostMemoryOverhead * memoryShare; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java index ccfe96d462c..1a6058bed39 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java @@ -30,8 +30,8 @@ public class SharedLoadBalancerServiceTest { public void test_create_lb() { LoadBalancerSpec spec = new LoadBalancerSpec(applicationId, clusterId, reals, ZoneEndpoint.defaultEndpoint, CloudAccount.empty); - loadBalancerService.provision(spec); - var lb = loadBalancerService.configure(spec, false); + + var lb = loadBalancerService.configure(loadBalancerService.provision(spec), spec, false); assertEquals(Optional.of(HostName.of("vip.example.com")), lb.hostname()); assertEquals(Optional.empty(), lb.dnsZone()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java index 1e1ccf37c8c..40ca30d758e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; @@ -62,10 +63,10 @@ public class AutoscalingMaintainerTest { tester.deploy(app1, cluster1, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), - IntRange.empty(), false, true, Optional.empty())); + IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty())); tester.deploy(app2, cluster2, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), new ClusterResources(10, 1, new NodeResources(6.5, 9, 20, 0.1)), - IntRange.empty(), false, true, Optional.empty())); + IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty())); tester.clock().advance(Duration.ofMinutes(10)); tester.maintainer().maintain(); // noop diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java index f8ec271ce5f..606bc55fdd2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java @@ -156,8 +156,8 @@ public class CapacityCheckerTester { .collect(Collectors.toSet()); NodeResources nr = containingNodeResources(childResources, excessCapacity); - Node node = Node.create(hostname, IP.Config.of(Set.of("::"), availableIps), hostname, - new Flavor(nr), NodeType.host).build(); + Node node = Node.create(hostname, new IP.Config(Set.of("::"), availableIps), hostname, + new Flavor(nr), NodeType.host).build(); hosts.computeIfAbsent(tenantHostApp, (ignored) -> new ArrayList<>()) .add(node); } @@ -175,8 +175,8 @@ public class CapacityCheckerTester { Set<String> availableIps = IntStream.range(2000, 2000 + ips) .mapToObj(n -> String.format("%04X::%04X", hostId, n)) .collect(Collectors.toSet()); - Node node = Node.create(hostname, IP.Config.of(Set.of("::" + (1000 + hostId)), availableIps), hostname, - new Flavor(capacity), NodeType.host).build(); + Node node = Node.create(hostname, new IP.Config(Set.of("::" + (1000 + hostId)), availableIps), hostname, + new Flavor(capacity), NodeType.host).build(); hosts.add(node); } return hosts; @@ -290,7 +290,7 @@ public class CapacityCheckerTester { Flavor f = new Flavor(nr); Node.Builder builder = Node.create(nodeModel.id, nodeModel.hostname, f, nodeModel.state, nodeModel.type) - .ipConfig(IP.Config.of(nodeModel.ipAddresses, nodeModel.additionalIpAddresses)); + .ipConfig(new IP.Config(nodeModel.ipAddresses, nodeModel.additionalIpAddresses)); nodeModel.parentHostname.ifPresent(builder::parentHostname); Node node = builder.build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java index e6d056d126d..880a69b61e5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; @@ -26,6 +27,7 @@ import com.yahoo.vespa.flags.custom.ClusterCapacity; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.Generation; @@ -122,7 +124,7 @@ public class HostCapacityMaintainerTest { public void preprovision_with_shared_host() { var tester = new DynamicProvisioningTester().addInitialNodes(); // Makes provisioned hosts 48-128-1000-10 - tester.hostProvisioner.overrideHostFlavor("host4"); + tester.hostProvisioner.setHostFlavor("host4"); var clusterCapacity = new ClusterCapacity(2, 1.0, 30.0, 20.0, 3.0, "fast", "local", "x86_64"); tester.flagSource.withListFlag(PermanentFlags.PREPROVISION_CAPACITY.id(), List.of(clusterCapacity), @@ -235,7 +237,7 @@ public class HostCapacityMaintainerTest { // Pretend shared-host flag has been set to host4's flavor var sharedHostNodeResources = new NodeResources(48, 128, 1000, 10, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote); - tester.hostProvisioner.overrideHostFlavor("host4"); + tester.hostProvisioner.setHostFlavor("host4"); // Next maintenance run does nothing tester.assertNodesUnchanged(); @@ -365,7 +367,7 @@ public class HostCapacityMaintainerTest { Cloud cloud = Cloud.builder().dynamicProvisioning(true).build(); DynamicProvisioningTester dynamicProvisioningTester = new DynamicProvisioningTester(cloud, new MockNameResolver().mockAnyLookup()); ProvisioningTester tester = dynamicProvisioningTester.provisioningTester; - dynamicProvisioningTester.hostProvisioner.overrideHostFlavor("default"); + dynamicProvisioningTester.hostProvisioner.setHostFlavor("default"); // Initial config server hosts are provisioned manually List<Node> provisionedHosts = tester.makeReadyNodes(3, "default", hostType, 1).stream() @@ -466,7 +468,7 @@ public class HostCapacityMaintainerTest { ClusterSpec spec = ProvisioningTester.contentClusterSpec(); ClusterResources resources = new ClusterResources(2, 1, new NodeResources(16, 24, 100, 1)); CloudAccount cloudAccount0 = CloudAccount.from("000000000000"); - Capacity capacity0 = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount0)); + Capacity capacity0 = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount0), ClusterInfo.empty()); List<HostSpec> prepared = provisioningTester.prepare(applicationId, spec, capacity0); // Hosts are provisioned in requested account @@ -476,7 +478,7 @@ public class HostCapacityMaintainerTest { // Redeployment in different account provisions a new set of hosts CloudAccount cloudAccount1 = CloudAccount.from("100000000000"); - Capacity capacity1 = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount1)); + Capacity capacity1 = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount1), ClusterInfo.empty()); prepared = provisioningTester.prepare(applicationId, spec, capacity1); provisionHostsIn(cloudAccount1, 2, tester); assertEquals(2, provisioningTester.activate(applicationId, prepared).size()); @@ -649,9 +651,9 @@ public class HostCapacityMaintainerTest { flavor.resources(), Generation.initial(), false)); - List<com.yahoo.config.provision.HostName> hostnames = Stream.of(additionalHostnames).map(com.yahoo.config.provision.HostName::of).toList(); + List<Address> addresses = Stream.of(additionalHostnames).map(Address::new).toList(); Node.Builder builder = Node.create("fake-id-" + hostname, hostname, flavor, state, nodeType) - .ipConfig(IP.Config.of(state == Node.State.active ? Set.of("::1") : Set.of(), Set.of(), hostnames)); + .ipConfig(new IP.Config(state == Node.State.active ? Set.of("::1") : Set.of(), Set.of(), addresses)); parentHostname.ifPresent(builder::parentHostname); allocation.ifPresent(builder::allocation); if (hostname.equals("host2-1")) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisionerTest.java index 8280c0e33fc..5e507d447ab 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisionerTest.java @@ -78,12 +78,12 @@ public class HostResumeProvisionerTest { hostResumeProvisioner.maintain(); assertTrue("No IP addresses written as DNS updates are failing", - provisioning.get().stream().allMatch(host -> host.ipConfig().pool().asSet().isEmpty())); + provisioning.get().stream().allMatch(host -> host.ipConfig().pool().ipSet().isEmpty())); hostProvisioner.without(MockHostProvisioner.Behaviour.failDnsUpdate); hostResumeProvisioner.maintain(); assertTrue("IP addresses written as DNS updates are succeeding", - provisioning.get().stream().noneMatch(host -> host.ipConfig().pool().asSet().isEmpty())); + provisioning.get().stream().noneMatch(host -> host.ipConfig().pool().ipSet().isEmpty())); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 2187611b702..d7ffda542ff 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -87,7 +87,7 @@ public class MetricsReporterTest { Map<String, Number> expectedMetrics = new TreeMap<>(); expectedMetrics.put("zone.working", 1); - expectedMetrics.put("hostedVespa.provisionedHosts", 1); + expectedMetrics.put("hostedVespa.provisionedHosts", 0); expectedMetrics.put("hostedVespa.parkedHosts", 0); expectedMetrics.put("hostedVespa.readyHosts", 0); expectedMetrics.put("hostedVespa.reservedHosts", 0); @@ -97,6 +97,16 @@ public class MetricsReporterTest { expectedMetrics.put("hostedVespa.failedHosts", 0); expectedMetrics.put("hostedVespa.deprovisionedHosts", 0); expectedMetrics.put("hostedVespa.breakfixedHosts", 0); + expectedMetrics.put("hostedVespa.provisionedNodes", 1); + expectedMetrics.put("hostedVespa.parkedNodes", 0); + expectedMetrics.put("hostedVespa.readyNodes", 0); + expectedMetrics.put("hostedVespa.reservedNodes", 0); + expectedMetrics.put("hostedVespa.activeNodes", 0); + expectedMetrics.put("hostedVespa.inactiveNodes", 0); + expectedMetrics.put("hostedVespa.dirtyNodes", 0); + expectedMetrics.put("hostedVespa.failedNodes", 0); + expectedMetrics.put("hostedVespa.deprovisionedNodes", 0); + expectedMetrics.put("hostedVespa.breakfixedNodes", 0); expectedMetrics.put("hostedVespa.pendingRedeployments", 42); expectedMetrics.put("hostedVespa.docker.totalCapacityDisk", 0.0); expectedMetrics.put("hostedVespa.docker.totalCapacityMem", 0.0); @@ -176,7 +186,7 @@ public class MetricsReporterTest { // Allow 4 containers Set<String> ipAddressPool = Set.of("::2", "::3", "::4", "::5"); - Node dockerHost = Node.create("node-id-1", IP.Config.of(Set.of("::1"), ipAddressPool), "dockerHost", + Node dockerHost = Node.create("node-id-1", new IP.Config(Set.of("::1"), ipAddressPool), "dockerHost", nodeFlavors.getFlavorOrThrow("host"), NodeType.host).build(); nodeRepository.nodes().addNodes(List.of(dockerHost), Agent.system); nodeRepository.nodes().deallocateRecursively("dockerHost", Agent.system, getClass().getSimpleName()); @@ -207,8 +217,8 @@ public class MetricsReporterTest { MetricsReporter metricsReporter = metricsReporter(metric, tester); metricsReporter.maintain(); - assertEquals(0, metric.values.get("hostedVespa.readyHosts")); // Only tenants counts - assertEquals(2, metric.values.get("hostedVespa.reservedHosts")); + assertEquals(0, metric.values.get("hostedVespa.readyNodes")); // Only tenants counts + assertEquals(2, metric.values.get("hostedVespa.reservedNodes")); assertEquals(120.0, metric.values.get("hostedVespa.docker.totalCapacityDisk")); assertEquals(100.0, metric.values.get("hostedVespa.docker.totalCapacityMem")); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index a07a4f2c72a..1b0826a8323 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -244,8 +244,8 @@ public class NodeFailTester { for (int i = startIndex; i < startIndex + count; i++) { String hostname = "host" + i; Set<String> ipPool = nodeType.isHost() ? Set.of("127.0." + i + "." + (++lastOctetOfPoolAddress)) : Set.of(); - IP.Config ipConfig = IP.Config.of(nodeRepository.nameResolver().resolveAll(hostname), - ipPool); + IP.Config ipConfig = new IP.Config(nodeRepository.nameResolver().resolveAll(hostname), + ipPool); Node.Builder builder = Node.create("node" + i, ipConfig, hostname, flavor, nodeType); parentHostname.ifPresent(builder::parentHostname); nodes.add(builder.build()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java index 47803594148..491485b78fc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java @@ -177,17 +177,17 @@ public class NodeFailerTest { @Test public void zone_is_not_working_if_too_many_nodes_down() { - NodeFailTester tester = NodeFailTester.withTwoApplications(); - - tester.serviceMonitor.setHostDown(tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app1).asList().get(0).hostname()); - tester.runMaintainers(); - assertTrue(tester.nodeRepository.nodes().isWorking()); + NodeFailTester tester = NodeFailTester.withTwoApplications(10, 5, 5); - tester.serviceMonitor.setHostDown(tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app1).asList().get(1).hostname()); - tester.runMaintainers(); - assertTrue(tester.nodeRepository.nodes().isWorking()); + int i = 0; + while (i < 4) { + tester.serviceMonitor.setHostDown(tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app1).asList().get(i).hostname()); + tester.runMaintainers(); + assertTrue(tester.nodeRepository.nodes().isWorking()); + i++; + } - tester.serviceMonitor.setHostDown(tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app1).asList().get(2).hostname()); + tester.serviceMonitor.setHostDown(tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app1).asList().get(i).hostname()); tester.runMaintainers(); assertFalse(tester.nodeRepository.nodes().isWorking()); @@ -199,6 +199,11 @@ public class NodeFailerTest { @Test public void node_failing() { NodeFailTester tester = NodeFailTester.withTwoApplications(6); + String downHost1 = tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app1).asList().get(1).hostname(); + String downHost2 = tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app2).asList().get(3).hostname(); + // No liveness evidence yet: + assertFalse(tester.nodeRepository.nodes().node(downHost1).get().isDown()); + assertFalse(tester.nodeRepository.nodes().node(downHost1).get().isUp()); // For a day all nodes work so nothing happens for (int minutes = 0; minutes < 24 * 60; minutes +=5 ) { @@ -208,10 +213,10 @@ public class NodeFailerTest { assertEquals(0, tester.deployer.redeployments); assertEquals(8, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.tenant).size()); assertEquals(0, tester.nodeRepository.nodes().list(Node.State.failed).nodeType(NodeType.tenant).size()); + assertFalse(tester.nodeRepository.nodes().node(downHost1).get().isDown()); + assertTrue(tester.nodeRepository.nodes().node(downHost1).get().isUp()); } - String downHost1 = tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app1).asList().get(1).hostname(); - String downHost2 = tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app2).asList().get(3).hostname(); tester.serviceMonitor.setHostDown(downHost1); tester.serviceMonitor.setHostDown(downHost2); // nothing happens the first 45 minutes @@ -221,12 +226,16 @@ public class NodeFailerTest { assertEquals(0, tester.deployer.redeployments); assertEquals(8, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.tenant).size()); assertEquals(0, tester.nodeRepository.nodes().list(Node.State.failed).nodeType(NodeType.tenant).size()); + assertTrue(tester.nodeRepository.nodes().node(downHost1).get().isDown()); + assertFalse(tester.nodeRepository.nodes().node(downHost1).get().isUp()); } tester.serviceMonitor.setHostUp(downHost1); // downHost2 should now be failed and replaced, but not downHost1 tester.clock.advance(Duration.ofDays(1)); tester.runMaintainers(); + assertFalse(tester.nodeRepository.nodes().node(downHost1).get().isDown()); + assertTrue(tester.nodeRepository.nodes().node(downHost1).get().isUp()); assertEquals(1, tester.deployer.redeployments); assertEquals(8, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.tenant).size()); assertEquals(1, tester.nodeRepository.nodes().list(Node.State.failed).nodeType(NodeType.tenant).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java index d379513a8f9..c7c6e770fe3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** + * @author bratseth */ public class NodeMetricsDbMaintainerTest { @@ -56,53 +57,56 @@ public class NodeMetricsDbMaintainerTest { private static class MockHttpClient implements MetricsV2MetricsFetcher.AsyncHttpClient { + // this value asserted on above final String cannedResponse = - "{\n" + - " \"nodes\": [\n" + - " {\n" + - " \"hostname\": \"host-1.yahoo.com\",\n" + - " \"role\": \"role0\",\n" + - " \"node\": {\n" + - " \"timestamp\": 1300,\n" + - " \"metrics\": [\n" + - " {\n" + - " \"values\": {\n" + - " \"cpu.util\": 14,\n" + // this value asserted on above - " \"mem_total.util\": 15,\n" + - " \"disk.util\": 20,\n" + - " \"application_generation\": 3,\n" + - " \"in_service\": 1\n" + - " },\n" + - " \"dimensions\": {\n" + - " \"state\": \"active\"\n" + - " }\n" + - " }\n" + - " ]\n" + - " }\n" + - " },\n" + - " {\n" + - " \"hostname\": \"host-2.yahoo.com\",\n" + - " \"role\": \"role0\",\n" + - " \"node\": {\n" + - " \"timestamp\": 1300,\n" + - " \"metrics\": [\n" + - " {\n" + - " \"values\": {\n" + - " \"cpu.util\": 1,\n" + - " \"mem_total.util\": 2,\n" + - " \"disk.util\": 3,\n" + - " \"application_generation\": 3,\n" + - " \"in_service\": 0\n" + - " },\n" + - " \"dimensions\": {\n" + - " \"state\": \"active\"\n" + - " }\n" + - " }\n" + - " ]\n" + - " }\n" + - " }\n" + - " ]\n" + - "}\n"; + """ + { + "nodes": [ + { + "hostname": "host-1.yahoo.com", + "role": "role0", + "node": { + "timestamp": 1300, + "metrics": [ + { + "values": { + "cpu.util": 14, + "mem_total.util": 15, + "disk.util": 20, + "application_generation.last": 3, + "in_service.last": 1 + }, + "dimensions": { + "state": "active" + } + } + ] + } + }, + { + "hostname": "host-2.yahoo.com", + "role": "role0", + "node": { + "timestamp": 1300, + "metrics": [ + { + "values": { + "cpu.util": 1, + "mem_total.util": 2, + "disk.util": 3, + "application_generation.last": 3, + "in_service.last": 0 + }, + "dimensions": { + "state": "active" + } + } + ] + } + } + ] + } + """; @Override public CompletableFuture<String> get(String url) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index 58f4be18992..f5e524a90cc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -239,8 +239,8 @@ public class RetiredExpirerTest { MockNameResolver nameResolver = (MockNameResolver) tester.nodeRepository().nameResolver(); String ipv4 = "127.0.1.4"; nameResolver.addRecord(retiredNode.hostname(), ipv4); - Node node = Node.create(retiredNode.hostname(), IP.Config.of(Set.of(ipv4), Set.of()), retiredNode.hostname(), - tester.asFlavor("default", NodeType.config), NodeType.config).build(); + Node node = Node.create(retiredNode.hostname(), new IP.Config(Set.of(ipv4), Set.of()), retiredNode.hostname(), + tester.asFlavor("default", NodeType.config), NodeType.config).build(); var nodes = List.of(node); nodes = nodeRepository.nodes().addNodes(nodes, Agent.system); nodes = nodeRepository.nodes().deallocate(nodes, Agent.system, getClass().getSimpleName()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java index 9c69543a9d6..2786da4b69e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.collections.Pair; import com.yahoo.config.provision.ApplicationId; @@ -56,10 +57,10 @@ public class ScalingSuggestionsMaintainerTest { tester.deploy(app1, cluster1, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), - IntRange.empty(), false, true, Optional.empty())); + IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty())); tester.deploy(app2, cluster2, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), new ClusterResources(10, 1, new NodeResources(6.5, 5, 15, 0.1)), - IntRange.empty(), false, true, Optional.empty())); + IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty())); tester.clock().advance(Duration.ofHours(13)); Duration timeAdded = addMeasurements(0.90f, 0.90f, 0.90f, 0, 500, app1, tester.nodeRepository()); @@ -97,7 +98,7 @@ public class ScalingSuggestionsMaintainerTest { maintainer.maintain(); var suggested = tester.nodeRepository().applications().get(app1).get().cluster(cluster1.id()).get().suggested().resources().get(); tester.deploy(app1, cluster1, Capacity.from(suggested, suggested, - IntRange.empty(), false, true, Optional.empty())); + IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty())); tester.clock().advance(Duration.ofDays(2)); addMeasurements(0.2f, 0.65f, 0.6f, 0, 500, app1, tester.nodeRepository()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java index b54975cbf41..c9421f098e7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java @@ -327,8 +327,8 @@ public class SpareCapacityMaintainerTest { } private IP.Config ipConfig(int id, boolean host) { - return IP.Config.of(Set.of(String.format("%04X::%04X", id, 0)), - host ? IntStream.range(0, 10) + return new IP.Config(Set.of(String.format("%04X::%04X", id, 0)), + host ? IntStream.range(0, 10) .mapToObj(n -> String.format("%04X::%04X", id, n)) .collect(Collectors.toSet()) : Set.of()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java index 88fe88dbaad..c26ffdaa023 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.provision.node; import com.google.common.collect.ImmutableSet; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.LockedNodeList; @@ -15,6 +14,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -156,7 +156,7 @@ public class IPTest { IP.Config config = IP.Config.of(Set.of("2600:1f10:::1"), Set.of("2600:1f10:::2", "2600:1f10:::3"), - List.of(HostName.of("node1"), HostName.of("node2"))); + List.of(new Address("node1"), new Address("node2"))); IP.Pool pool = config.pool(); Optional<IP.Allocation> allocation = pool.findAllocation(emptyList, resolver, false); } @@ -193,12 +193,12 @@ public class IPTest { } IP.Pool pool = node.ipConfig().pool(); - assertNotEquals(dualStack, pool.ipAddresses().protocol() == IP.IpAddresses.Protocol.ipv4); + assertNotEquals(dualStack, pool.getProtocol() == IP.IpAddresses.Protocol.ipv4); return pool; } private static Node createNode(Set<String> ipAddresses) { - return Node.create("id1", IP.Config.of(Set.of("127.0.0.1"), ipAddresses), + return Node.create("id1", new IP.Config(Set.of("127.0.0.1"), ipAddresses), "host1", nodeFlavors.getFlavorOrThrow("default"), NodeType.host).build(); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java index d04fe1bdda2..c429f88cfa1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterResources; @@ -15,6 +16,7 @@ import com.yahoo.vespa.hosted.provision.autoscale.Autoscaling; import com.yahoo.vespa.hosted.provision.autoscale.Load; import org.junit.Test; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -40,6 +42,7 @@ public class ApplicationSerializerTest { true, Autoscaling.empty(), Autoscaling.empty(), + ClusterInfo.empty(), BcpGroupInfo.empty(), List.of())); var minResources = new NodeResources(1, 2, 3, 4); @@ -65,6 +68,7 @@ public class ApplicationSerializerTest { Load.zero(), Load.one(), Autoscaling.Metrics.zero()), + new ClusterInfo.Builder().bcpDeadline(Duration.ofMinutes(33)).build(), new BcpGroupInfo(0.1, 0.2, 0.3), List.of(new ScalingEvent(new ClusterResources(10, 5, minResources), new ClusterResources(12, 6, minResources), @@ -95,6 +99,7 @@ public class ApplicationSerializerTest { assertEquals(originalCluster.required(), serializedCluster.required()); assertEquals(originalCluster.suggested(), serializedCluster.suggested()); assertEquals(originalCluster.target(), serializedCluster.target()); + assertEquals(originalCluster.clusterInfo(), serializedCluster.clusterInfo()); assertEquals(originalCluster.bcpGroupInfo(), serializedCluster.bcpGroupInfo()); assertEquals(originalCluster.scalingEvents(), serializedCluster.scalingEvents()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ArchiveUriSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ArchiveUriSerializerTest.java new file mode 100644 index 00000000000..6204ace8a51 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ArchiveUriSerializerTest.java @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.provision.archive.ArchiveUris; +import org.junit.Test; + +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * @author freva + */ +public class ArchiveUriSerializerTest { + + @Test + public void test_serialization() { + ArchiveUris archiveUris = new ArchiveUris(Map.of( + TenantName.from("tenant1"), "ftp://host123.test/dir/", + TenantName.from("tenant2"), "ftp://archive.test/vespa/"), + Map.of(CloudAccount.from("321456987012"), "ftp://host123.test/dir/")); + + ArchiveUris serialized = ArchiveUriSerializer.fromJson(ArchiveUriSerializer.toJson(archiveUris)); + assertEquals(archiveUris, serialized); + } + +}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index 1086f2026a8..d61a3d95a65 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NetworkPorts; import com.yahoo.config.provision.NodeFlavors; @@ -24,6 +23,7 @@ import com.yahoo.test.ManualClock; import com.yahoo.text.Utf8; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.Node.State; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Generation; import com.yahoo.vespa.hosted.provision.node.History; @@ -249,7 +249,7 @@ public class NodeSerializerTest { @Test public void serialize_parent_hostname() { final String parentHostname = "parent.yahoo.com"; - Node node = Node.create("myId", IP.Config.of(Set.of("127.0.0.1"), Set.of()), "myHostname", nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant) + Node node = Node.create("myId", new IP.Config(Set.of("127.0.0.1"), Set.of()), "myHostname", nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant) .parentHostname(parentHostname) .build(); @@ -272,14 +272,16 @@ public class NodeSerializerTest { // Test round-trip with address pool node = node.with(node.ipConfig().withPool(IP.Pool.of( Set.of("::1", "::2", "::3"), - List.of(HostName.of("a"), HostName.of("b"), HostName.of("c"))))); + List.of(new Address("a"), new Address("b"), new Address("c"))))); Node copy = nodeSerializer.fromJson(nodeSerializer.toJson(node)); - assertEquals(node.ipConfig(), copy.ipConfig()); + assertEquals(node.ipConfig().pool().ipSet(), copy.ipConfig().pool().ipSet()); + assertEquals(Set.copyOf(node.ipConfig().pool().getAddressList()), Set.copyOf(copy.ipConfig().pool().getAddressList())); // Test round-trip without address pool (handle empty pool) node = createNode(); copy = nodeSerializer.fromJson(nodeSerializer.toJson(node)); - assertEquals(node.ipConfig(), copy.ipConfig()); + assertEquals(node.ipConfig().pool().ipSet(), copy.ipConfig().pool().ipSet()); + assertEquals(Set.copyOf(node.ipConfig().pool().getAddressList()), Set.copyOf(copy.ipConfig().pool().getAddressList())); } @Test @@ -527,7 +529,7 @@ public class NodeSerializerTest { private Node createNode() { return Node.create("myId", - IP.Config.of(Set.of("127.0.0.1"), Set.of()), + new IP.Config(Set.of("127.0.0.1"), Set.of()), "myHostname", nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant).build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/TenantArchiveUriSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/TenantArchiveUriSerializerTest.java deleted file mode 100644 index 2ae4f6363e0..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/TenantArchiveUriSerializerTest.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.persistence; - -import com.yahoo.config.provision.TenantName; -import org.junit.Test; - -import java.util.Map; -import java.util.TreeMap; - -import static org.junit.Assert.assertEquals; - -/** - * @author freva - */ -public class TenantArchiveUriSerializerTest { - - @Test - public void test_serialization() { - Map<TenantName, String> archiveUris = new TreeMap<>(); - archiveUris.put(TenantName.from("tenant1"), "ftp://host123.test/dir/"); - archiveUris.put(TenantName.from("tenant2"), "ftp://archive.test/vespa/"); - - Map<TenantName, String> serialized = TenantArchiveUriSerializer.fromJson(TenantArchiveUriSerializer.toJson(archiveUris)); - assertEquals(archiveUris, serialized); - } - -}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ArchiveUrisTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ArchiveUrisTest.java deleted file mode 100644 index 7751e906c48..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ArchiveUrisTest.java +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.provisioning; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterMembership; -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.node.Allocation; -import com.yahoo.vespa.hosted.provision.node.Generation; -import org.junit.Test; - -import java.util.Optional; - -import static com.yahoo.vespa.hosted.provision.provisioning.ArchiveUris.normalizeUri; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.fail; - -/** - * @author freva - */ -public class ArchiveUrisTest { - - @Test - public void archive_uri() { - ApplicationId app = ApplicationId.from("vespa", "music", "main"); - Node allocated = createNode(app); - Node unallocated = createNode(null); - ArchiveUris archiveUris = new ProvisioningTester.Builder().build().nodeRepository().archiveUris(); - - assertFalse(archiveUris.archiveUriFor(unallocated).isPresent()); - assertFalse(archiveUris.archiveUriFor(allocated).isPresent()); - - archiveUris.setArchiveUri(app.tenant(), Optional.of("scheme://hostname/dir")); - assertEquals("scheme://hostname/dir/music/main/default/h432a/", archiveUris.archiveUriFor(allocated).get()); - } - - private Node createNode(ApplicationId appId) { - Node.Builder nodeBuilder = Node.create("id", "h432a.prod.us-south-1.vespa.domain.tld", new Flavor(NodeResources.unspecified()), Node.State.parked, NodeType.tenant); - Optional.ofNullable(appId) - .map(app -> new Allocation(app, - ClusterMembership.from("container/default/0/0", Version.fromString("1.2.3"), Optional.empty()), - NodeResources.unspecified(), - Generation.initial(), - false)) - .ifPresent(nodeBuilder::allocation); - return nodeBuilder.build(); - } - - @Test - public void normalize_test() { - assertEquals("ftp://domain/legal-dir123/", normalizeUri("ftp://domain/legal-dir123")); - assertEquals("ftp://domain/legal-dir123/", normalizeUri("ftp://domain/legal-dir123/")); - assertEquals("s3://my-bucket-prod.region/my-tenant-123/", normalizeUri("s3://my-bucket-prod.region/my-tenant-123/")); - assertEquals("s3://my-bucket-prod.region/my-tenant_123/", normalizeUri("s3://my-bucket-prod.region/my-tenant_123/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("domain/dir/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp:/domain/dir/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp:/domain//dir/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/illegal:dir/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/-illegal-dir/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/_illegal-dir/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/illegal-dir-/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/illegal-dir_/")); - } - - private static void assertThrows(Class<? extends Throwable> clazz, Runnable runnable) { - try { - runnable.run(); - fail("Expected " + clazz); - } catch (Throwable e) { - if (!clazz.isInstance(e)) throw e; - } - } -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java index 47d34a76dd6..23c2d0fc47a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java @@ -534,7 +534,7 @@ public class DynamicAllocationTest { } private void addAndAssignNode(ApplicationId id, String hostname, String parentHostname, ClusterSpec clusterSpec, NodeResources flavor, int index, ProvisioningTester tester) { - Node node1a = Node.create("open1", IP.Config.of(Set.of("127.0.233." + index), Set.of()), hostname, + Node node1a = Node.create("open1", new IP.Config(Set.of("127.0.233." + index), Set.of()), hostname, new Flavor(flavor), NodeType.tenant).parentHostname(parentHostname).build(); ClusterMembership clusterMembership1 = ClusterMembership.from( clusterSpec.with(Optional.of(ClusterSpec.Group.from(0))), index); // Need to add group here so that group is serialized in node allocation diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java index f406f44f02f..0e19c48591d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java @@ -256,13 +256,13 @@ public class DynamicProvisioningTest { ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("8").build(); Capacity capacity = Capacity.from(new ClusterResources(4, 2, new NodeResources(2, 4, 50, 0.1, DiskSpeed.any, StorageType.any, Architecture.any))); - hostProvisioner.overrideHostFlavor("x86"); + hostProvisioner.setHostFlavor("x86", ClusterSpec.Type.content); tester.activate(app, cluster, capacity); NodeList nodes = tester.nodeRepository().nodes().list(); assertEquals(4, nodes.owner(app).state(Node.State.active).size()); assertEquals(Set.of("x86"), nodes.parentsOf(nodes.owner(app).state(Node.State.active)).stream().map(n -> n.flavor().name()).collect(Collectors.toSet())); - hostProvisioner.overrideHostFlavor("arm"); + hostProvisioner.setHostFlavor("arm", ClusterSpec.Type.content); flagSource.withStringFlag(PermanentFlags.HOST_FLAVOR.id(), "arm"); tester.activate(app, cluster, capacity); nodes = tester.nodeRepository().nodes().list(); @@ -271,7 +271,7 @@ public class DynamicProvisioningTest { assertEquals(Set.of("x86"), nodes.parentsOf(tester.getNodes(app, Node.State.active).retired()).stream().map(n -> n.flavor().name()).collect(Collectors.toSet())); assertEquals(Set.of("arm"), nodes.parentsOf(tester.getNodes(app, Node.State.active).not().retired()).stream().map(n -> n.flavor().name()).collect(Collectors.toSet())); - flagSource.removeFlag(PermanentFlags.HOST_FLAVOR.id()); // Resetting flag does not moves the nodes back + flagSource.removeFlag(PermanentFlags.HOST_FLAVOR.id()); // Resetting flag does not move the nodes back tester.activate(app, cluster, capacity); nodes = tester.nodeRepository().nodes().list(); assertEquals(4, nodes.owner(app).state(Node.State.active).retired().size()); @@ -453,7 +453,7 @@ public class DynamicProvisioningTest { if (!provisionedHosts.isEmpty()) { List<Node> hosts = provisionedHosts.asList() .stream() - .map(h -> h.with(((MockHostProvisioner) tester.hostProvisioner()).createIpConfig(h))) + .map(h -> ((MockHostProvisioner)tester.hostProvisioner()).withIpAssigned(h)) .toList(); tester.move(Node.State.ready, hosts); tester.activateTenantHosts(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java index dcdf79a3951..a24eb61bb79 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.autoscale; +package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; @@ -19,6 +19,10 @@ import com.yahoo.vespa.hosted.provision.Nodelike; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; +import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; +import com.yahoo.vespa.hosted.provision.autoscale.Autoscaling; +import com.yahoo.vespa.hosted.provision.autoscale.Fixture; +import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.CapacityPolicies; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; @@ -34,28 +38,43 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** + * A provisioniong tester which + * - Supports dynamic provisioning (only). + * - Optionally replicates the actual AWS setup and logic used on Vespa Cloud. + * - Supports autoscaling testing. + * + * TODO: All provisioning testing should migrate to use this, and then the provisionging tester should be collapsed + * into this. + * * @author bratseth */ -class AutoscalingTester { +public class DynamicProvisioningTester { private final ProvisioningTester provisioningTester; private final Autoscaler autoscaler; private final HostResourcesCalculator hostResourcesCalculator; private final CapacityPolicies capacityPolicies; - public AutoscalingTester(Zone zone, HostResourcesCalculator resourcesCalculator, List<Flavor> hostFlavors, InMemoryFlagSource flagSource, int hostCount) { + public DynamicProvisioningTester(Zone zone, HostResourcesCalculator resourcesCalculator, List<Flavor> hostFlavors, InMemoryFlagSource flagSource, int hostCount) { this(zone, hostFlavors, resourcesCalculator, flagSource); for (Flavor flavor : hostFlavors) provisioningTester.makeReadyNodes(hostCount, flavor.name(), NodeType.host, 8); provisioningTester.activateTenantHosts(); } - private AutoscalingTester(Zone zone, List<Flavor> flavors, HostResourcesCalculator resourcesCalculator, InMemoryFlagSource flagSource) { + private DynamicProvisioningTester(Zone zone, List<Flavor> flavors, HostResourcesCalculator resourcesCalculator, InMemoryFlagSource flagSource) { + MockHostProvisioner hostProvisioner = null; + if (zone.cloud().dynamicProvisioning()) { + hostProvisioner = new MockHostProvisioner(flavors); + hostProvisioner.setHostFlavorIfAvailable(new NodeResources(2, 8, 75, 10, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), resourcesCalculator, ClusterSpec.Type.admin + ); + } + provisioningTester = new ProvisioningTester.Builder().zone(zone) .flavors(flavors) .resourcesCalculator(resourcesCalculator) .flagSource(flagSource) - .hostProvisioner(zone.cloud().dynamicProvisioning() ? new MockHostProvisioner(flavors) : null) + .hostProvisioner(hostProvisioner) .build(); hostResourcesCalculator = resourcesCalculator; @@ -105,9 +124,9 @@ class AutoscalingTester { public void makeReady(String hostname) { Node node = nodeRepository().nodes().node(hostname).get(); - provisioningTester.patchNode(node, (n) -> n.with(IP.Config.of(Set.of("::" + 0 + ":0"), Set.of()))); + provisioningTester.patchNode(node, (n) -> n.with(new IP.Config(Set.of("::" + 0 + ":0"), Set.of()))); Node host = nodeRepository().nodes().node(node.parentHostname().get()).get(); - host = host.with(IP.Config.of(Set.of("::" + 0 + ":0"), Set.of("::" + 0 + ":2"))); + host = host.with(new IP.Config(Set.of("::" + 0 + ":0"), Set.of("::" + 0 + ":2"))); if (host.state() == Node.State.provisioned) provisioningTester.move(Node.State.ready, host); } @@ -134,6 +153,7 @@ class AutoscalingTester { cluster.required(), cluster.suggested(), cluster.target(), + cluster.clusterInfo(), cluster.bcpGroupInfo(), List.of()); // Remove scaling events cluster = cluster.with(ScalingEvent.create(cluster.minResources(), cluster.minResources(), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java index 12a8e4d9386..ea2af5f3fca 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java @@ -2,12 +2,12 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.IP; import org.junit.Before; import org.junit.Test; @@ -16,6 +16,7 @@ import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; import static org.junit.Assert.assertEquals; @@ -166,7 +167,7 @@ public class HostCapacityTest { } private Node setupHostWithAdditionalHostnames(String hostHostname, String... additionalHostnames) { - List<HostName> hostnames = Stream.of(additionalHostnames).map(HostName::of).toList(); + List<Address> addresses = Stream.of(additionalHostnames).map(Address::new).toList(); doAnswer(invocation -> ((Flavor)invocation.getArguments()[0]).resources()) .when(hostResourcesCalculator).advertisedResourcesOf(any()); @@ -175,7 +176,7 @@ public class HostCapacityTest { "host", // 7-100-120-5 "docker"); // 2- 40- 40-0.5 = resources1 - return Node.create(hostHostname, IP.Config.of(Set.of("::1"), Set.of(), hostnames), hostHostname, + return Node.create(hostHostname, IP.Config.of(Set.of("::1"), Set.of(), addresses), hostHostname, nodeFlavors.getFlavorOrThrow("host"), NodeType.host).build(); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index c791c7848d7..d7b5f30a9bc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -6,6 +6,7 @@ import com.google.common.collect.Iterators; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; @@ -326,7 +327,7 @@ public class LoadBalancerProvisionerTest { @Test public void load_balancer_with_custom_settings() { ClusterResources resources = new ClusterResources(3, 1, nodeResources); - Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(CloudAccount.empty)); + Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(CloudAccount.empty), ClusterInfo.empty()); tester.activate(app1, prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1")))); LoadBalancerList loadBalancers = tester.nodeRepository().loadBalancers().list(); assertEquals(1, loadBalancers.size()); @@ -343,7 +344,7 @@ public class LoadBalancerProvisionerTest { @Test public void load_balancer_with_changing_visibility() { ClusterResources resources = new ClusterResources(3, 1, nodeResources); - Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(CloudAccount.empty)); + Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(CloudAccount.empty), ClusterInfo.empty()); tester.activate(app1, prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1")))); LoadBalancerList loadBalancers = tester.nodeRepository().loadBalancers().list(); assertEquals(1, loadBalancers.size()); @@ -379,7 +380,7 @@ public class LoadBalancerProvisionerTest { ClusterResources resources = new ClusterResources(3, 1, nodeResources); CloudAccount cloudAccount0 = CloudAccount.empty; { - Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount0)); + Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount0), ClusterInfo.empty()); tester.activate(app1, prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1")))); } LoadBalancerList loadBalancers = tester.nodeRepository().loadBalancers().list(); @@ -388,7 +389,7 @@ public class LoadBalancerProvisionerTest { // Changing account fails if there is an existing LB in the previous account. CloudAccount cloudAccount1 = CloudAccount.from("111111111111"); - Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount1)); + Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount1), ClusterInfo.empty()); try { prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1"))); fail("Expected exception"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java index 32db213c445..20819daf356 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java @@ -146,7 +146,7 @@ public class NodeCandidateTest { .parentHostname(hostname + "parent") .ipConfigWithEmptyPool(Set.of("::1")).build(); Node parent = Node.create(hostname + "parent", hostname, new Flavor(totalHostResources), Node.State.ready, NodeType.host) - .ipConfig(IP.Config.of(Set.of("::1"), Set.of("::2"))) + .ipConfig(new IP.Config(Set.of("::1"), Set.of("::2"))) .build(); return new NodeCandidate.ConcreteNodeCandidate(node, totalHostResources.subtract(allocatedHostResources), Optional.of(parent), false, exclusiveSwitch, false, true, false); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 68857719bf0..4c404e3030c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -919,10 +919,10 @@ public class ProvisioningTest { // Add 2 config server hosts and 2 config servers Flavor flavor = tester.nodeRepository().flavors().getFlavorOrThrow("default"); List<Node> nodes = List.of( - Node.create("cfghost1", IP.Config.of(Set.of("::1:0"), Set.of("::1:1")), "cfghost1", flavor, NodeType.confighost).build(), - Node.create("cfghost2", IP.Config.of(Set.of("::2:0"), Set.of("::2:1")), "cfghost2", flavor, NodeType.confighost).ipConfig(IP.Config.of(Set.of("::2:0"), Set.of("::2:1"), List.of())).build(), - Node.create("cfg1", IP.Config.of(Set.of("::1:1"), Set.of()), "cfg1", flavor, NodeType.config).parentHostname("cfghost1").build(), - Node.create("cfg2", IP.Config.of(Set.of("::2:1"), Set.of()), "cfg2", flavor, NodeType.config).parentHostname("cfghost2").build()); + Node.create("cfghost1", new IP.Config(Set.of("::1:0"), Set.of("::1:1")), "cfghost1", flavor, NodeType.confighost).build(), + Node.create("cfghost2", new IP.Config(Set.of("::2:0"), Set.of("::2:1")), "cfghost2", flavor, NodeType.confighost).ipConfig(IP.Config.of(Set.of("::2:0"), Set.of("::2:1"), List.of())).build(), + Node.create("cfg1", new IP.Config(Set.of("::1:1"), Set.of()), "cfg1", flavor, NodeType.config).parentHostname("cfghost1").build(), + Node.create("cfg2", new IP.Config(Set.of("::2:1"), Set.of()), "cfg2", flavor, NodeType.config).parentHostname("cfghost2").build()); tester.move(Node.State.ready, tester.nodeRepository().nodes().addNodes(nodes, Agent.system)); InfraApplication cfgHostApp = new ConfigServerHostApplication(); @@ -1041,6 +1041,19 @@ public class ProvisioningTest { assertEquals(new NodeResources(3, 3, 3, 3), CapacityPolicies.versioned(spec.vespaVersion("9.0").build(), resources)); } + @Test + public void testAdminProvisioning() { + var nodeResources = new NodeResources(0.25, 1.32, 10, 0.3); + var resources = new ClusterResources(1, 1, nodeResources); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .clusterType(ClusterSpec.Type.admin) + .initialResources(Optional.empty()) + .capacity(Capacity.from(resources)) + .build(); + fixture.deploy(); + } + private SystemState prepare(ApplicationId application, int container0Size, int container1Size, int content0Size, int content1Size, NodeResources flavor, ProvisioningTester tester) { return prepare(application, tester, container0Size, container1Size, content0Size, content1Size, flavor, "6.42"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index d0ff11fde0c..06f26bc1d8b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -201,7 +201,7 @@ public class ProvisioningTester { try (var lock = nodeRepository.nodes().lockAndGetRequired(prepared.hostname())) { Node node = lock.node(); if (node.ipConfig().primary().isEmpty()) { - node = node.with(IP.Config.of(Set.of("::" + 0 + ":0"), Set.of())); + node = node.with(new IP.Config(Set.of("::" + 0 + ":0"), Set.of())); nodeRepository.nodes().write(node, lock); } if (node.parentHostname().isEmpty()) continue; @@ -209,7 +209,7 @@ public class ProvisioningTester { if (parent.state() == Node.State.active) continue; NestedTransaction t = new NestedTransaction(); if (parent.ipConfig().primary().isEmpty()) - parent = parent.with(IP.Config.of(Set.of("::" + 0 + ":0"), Set.of("::" + 0 + ":2"))); + parent = parent.with(new IP.Config(Set.of("::" + 0 + ":0"), Set.of("::" + 0 + ":2"))); nodeRepository.nodes().activate(List.of(parent), t); t.commit(); } @@ -447,7 +447,7 @@ public class ProvisioningTester { nameResolver.addRecord(nodeHostname, ipv4Addr); } } - Node.Builder builder = Node.create(hostname, IP.Config.of(hostIps, ipAddressPool), hostname, flavor, type); + Node.Builder builder = Node.create(hostname, new IP.Config(hostIps, ipAddressPool), hostname, flavor, type); reservedTo.ifPresent(builder::reservedTo); nodes.add(builder.build()); } @@ -464,8 +464,8 @@ public class ProvisioningTester { String ipv4 = "127.0.1." + i; nameResolver.addRecord(hostname, ipv4); - Node node = Node.create(hostname, IP.Config.of(Set.of(ipv4), Set.of()), hostname, - nodeFlavors.getFlavorOrThrow(flavor), NodeType.config).build(); + Node node = Node.create(hostname, new IP.Config(Set.of(ipv4), Set.of()), hostname, + nodeFlavors.getFlavorOrThrow(flavor), NodeType.config).build(); nodes.add(node); } @@ -532,7 +532,7 @@ public class ProvisioningTester { List<Node> nodes = new ArrayList<>(count); for (int i = startIndex; i < count + startIndex; i++) { String hostname = nodeNamer.apply(i); - IP.Config ipConfig = IP.Config.of(nodeRepository.nameResolver().resolveAll(hostname), Set.of()); + IP.Config ipConfig = new IP.Config(nodeRepository.nameResolver().resolveAll(hostname), Set.of()); Node node = Node.create("node-id", ipConfig, hostname, new Flavor(resources), nodeType) .parentHostname(parentHostname) .build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java new file mode 100644 index 00000000000..7fe2d77b647 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java @@ -0,0 +1,67 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.application.container.handler.Request; +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.SystemName; +import com.yahoo.text.Utf8; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +/** + * Test of the REST APIs provided by the node repository. + * + * Note: This class is referenced from our operations documentation and must not be renamed/moved without updating that. + * + * @author bratseth + */ +public class ArchiveApiTest { + + private RestApiTester tester; + + @Before + public void createTester() { + tester = new RestApiTester(SystemName.Public, CloudAccount.from("111222333444")); + } + + @After + public void closeTester() { + tester.close(); + } + + @Test + public void archive_uris() throws IOException { + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "archiveUri", false); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/archive"), "{\"archives\":[]}"); + + assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant/tenant3", Utf8.toBytes("{\"uri\": \"ftp://host/dir\"}"), Request.Method.PATCH), + "{\"message\":\"Updated archive URI for tenant3\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant/tenant2", Utf8.toBytes("{\"uri\": \"s3://my-bucket/dir\"}"), Request.Method.PATCH), + "{\"message\":\"Updated archive URI for tenant2\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/archive/account/777888999000", Utf8.toBytes("{\"uri\": \"s3://acc-bucket\"}"), Request.Method.PATCH), + "{\"message\":\"Updated archive URI for 777888999000\"}"); + + + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "\"archiveUri\":\"ftp://host/dir/tenant3/application3/instance3/id3/host4/\"", true); + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com"), "\"archiveUri\":\"s3://acc-bucket/zoneapp/zoneapp/zoneapp/node-admin/dockerhost2/\"", true); + assertFile(new Request("http://localhost:8080/nodes/v2/archive"), "archives.json"); + + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant/tenant3", new byte[0], Request.Method.DELETE), "{\"message\":\"Removed archive URI for tenant3\"}"); + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "archiveUri", false); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/archive/account/777888999000", new byte[0], Request.Method.DELETE), "{\"message\":\"Removed archive URI for 777888999000\"}"); + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com"), "archiveUri", false); + } + + + private void assertFile(Request request, String file) throws IOException { + tester.assertFile(request, file); + } + + private void assertResponse(Request request, String response) throws IOException { + tester.assertResponse(request, response); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java index 240d0daf96f..729b6b813cd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.SystemName; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -16,7 +17,7 @@ public class LoadBalancersV1ApiTest { @Before public void createTester() { - tester = new RestApiTester(CloudAccount.empty); + tester = new RestApiTester(SystemName.main, CloudAccount.empty); } @After diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 1d7b7ec7454..c9e57c22d11 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -6,6 +6,7 @@ import com.yahoo.application.container.handler.Response; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.text.Utf8; import com.yahoo.vespa.applicationmodel.HostName; @@ -41,7 +42,7 @@ public class NodesV2ApiTest { @Before public void createTester() { - tester = new RestApiTester(CloudAccount.from("111222333444")); + tester = new RestApiTester(SystemName.main, CloudAccount.from("111222333444")); } @After @@ -112,8 +113,8 @@ public class NodesV2ApiTest { // POST duplicate node tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.254.8") + "]").getBytes(StandardCharsets.UTF_8), - Request.Method.POST), 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot add provisioned host host8.yahoo.com: A node with this name already exists\"}"); + Request.Method.POST), 500, + "{\"error-code\":\"INTERNAL_SERVER_ERROR\",\"message\":\"Cannot add provisioned host host8.yahoo.com: A node with this name already exists\"}"); // DELETE a provisioned node assertResponse(new Request("http://localhost:8080/nodes/v2/node/host9.yahoo.com", @@ -990,23 +991,6 @@ public class NodesV2ApiTest { } @Test - public void archive_uris() throws IOException { - tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "archiveUri", false); - tester.assertResponse(new Request("http://localhost:8080/nodes/v2/archive"), "{\"archives\":[]}"); - - assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant3", Utf8.toBytes("{\"uri\": \"ftp://host/dir\"}"), Request.Method.PATCH), - "{\"message\":\"Updated archive URI for tenant3\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant2", Utf8.toBytes("{\"uri\": \"s3://my-bucket/dir\"}"), Request.Method.PATCH), - "{\"message\":\"Updated archive URI for tenant2\"}"); - - tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "\"archiveUri\":\"ftp://host/dir/application3/instance3/id3/host4/\"", true); - assertFile(new Request("http://localhost:8080/nodes/v2/archive"), "archives.json"); - - tester.assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant3", new byte[0], Request.Method.DELETE), "{\"message\":\"Removed archive URI for tenant3\"}"); - tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "archiveUri", false); - } - - @Test public void trusted_certificates_patch() throws IOException { String url = "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"; tester.assertPartialResponse(new Request(url), "\"trustStore\":[]", false); // initially empty list diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java index e424b04aeaf..47745d25467 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java @@ -6,6 +6,7 @@ import com.yahoo.application.container.JDisc; import com.yahoo.application.container.handler.Request; import com.yahoo.application.container.handler.Response; import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.SystemName; import com.yahoo.io.IOUtils; import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; import org.junit.ComparisonFailure; @@ -25,8 +26,8 @@ public class RestApiTester { private final JDisc container; - public RestApiTester(CloudAccount defaultCloudAccount) { - container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0, defaultCloudAccount), Networking.disable); + public RestApiTester(SystemName systemName, CloudAccount defaultCloudAccount) { + container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0, systemName, defaultCloudAccount), Networking.disable); } public void close() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/archives.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/archives.json index 1ce54b54f6a..738d8ee1bb3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/archives.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/archives.json @@ -7,6 +7,10 @@ { "tenant": "tenant3", "uri": "ftp://host/dir/" + }, + { + "account": "777888999000", + "uri": "s3://acc-bucket/" } ] } diff --git a/parent/pom.xml b/parent/pom.xml index 2fcf5527302..fb6b0abf395 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -468,6 +468,11 @@ <version>3.10.0</version> </dependency> <dependency> + <groupId>com.fasterxml.jackson.dataformat</groupId> + <artifactId>jackson-dataformat-cbor</artifactId> + <version>${jackson2.version}</version> + </dependency> + <dependency> <groupId>com.github.cverges.expect4j</groupId> <artifactId>expect4j</artifactId> <version>1.6</version> @@ -1128,7 +1133,7 @@ xargs perl -pi -e 's/major = [0-9]+, minor = [0-9]+, micro = [0-9]+/major = 5, minor = 3, micro = 0/g' --> <bouncycastle.version>1.72</bouncycastle.version> - <curator.version>5.3.0</curator.version> + <curator.version>5.4.0</curator.version> <commons-codec.version>1.15</commons-codec.version> <commons-io.version>2.11.0</commons-io.version> <commons.math3.version>3.6.1</commons.math3.version> @@ -1170,7 +1175,7 @@ <prometheus.client.version>0.6.0</prometheus.client.version> <protobuf.version>3.21.7</protobuf.version> <spifly.version>1.3.5</spifly.version> - <surefire.version>2.22.2</surefire.version> + <surefire.version>3.0.0-M9</surefire.version> <wiremock.version>2.35.0</wiremock.version> <zero-allocation-hashing.version>0.16</zero-allocation-hashing.version> <zookeeper.client.version>3.8.0</zookeeper.client.version> diff --git a/persistence/CMakeLists.txt b/persistence/CMakeLists.txt index d87f172bfb5..072e273338b 100644 --- a/persistence/CMakeLists.txt +++ b/persistence/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib document @@ -27,7 +27,6 @@ <module>airlift-zstd</module> <module>application</module> <module>application-model</module> - <module>athenz-identity-provider-service</module> <module>bundle-plugin-test</module> <module>client</module> <module>cloud-tenant-base</module> diff --git a/screwdriver.yaml b/screwdriver.yaml index d918078b80e..48dfac2e644 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -220,8 +220,9 @@ jobs: VESPA_REF=$(meta get vespa.ref) if [[ $VESPA_VERSION == null ]] || [[ $VESPA_REF == null ]]; then echo "Must have valid Vespa version and reference to continue (got VESPA_VERSION=$VESPA_VERSION, VESPA_REF=$VESPA_REF)." - exit 1 + return 1 fi + meta set vespa.version $VESPA_VERSION - install-dependencies: | dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo dnf install -y docker-ce docker-ce-cli containerd.io @@ -235,6 +236,95 @@ jobs: - update-sample-apps: | screwdriver/update-vespa-version-in-sample-apps.sh $VESPA_VERSION + publish-legacy-release: + image: docker.io/vespaengine/vespa-build-centos-stream8:latest + + annotations: + screwdriver.cd/cpu: 7 + screwdriver.cd/ram: 16 + screwdriver.cd/disk: HIGH + screwdriver.cd/timeout: 300 + screwdriver.cd/dockerEnabled: true + screwdriver.cd/dockerCpu: TURBO + screwdriver.cd/dockerRam: HIGH + screwdriver.cd/buildPeriodically: H 6 1 * * + + environment: + IMAGE_NAME: "vespaengine/vespa-generic-intel-x86_64" + + secrets: + - DOCKER_HUB_DEPLOY_KEY + + steps: + - get-vespa-version: | + set -x + VESPA_VERSION=$(meta get vespa.version --external publish-release) + if [[ $VESPA_VERSION == null ]] || [[ $VESPA_REF == null ]]; then + echo "Must have valid Vespa version to continue (got VESPA_VERSION=$VESPA_VERSION)." + return 1 + fi + - install-dependencies: | + dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + dnf install -y docker-ce docker-ce-cli containerd.io + docker system info + - checkout: | + mkdir -p workdir + cd workdir + export WORKDIR=$(pwd) + git clone -q https://github.com/vespa-engine/vespa + (cd vespa && git checkout v$VESPA_VERSION) + git clone -q https://github.com/vespa-engine/system-test + # Set correct version in pom.xml files + (cd vespa && screwdriver/replace-vespa-version-in-poms.sh $VESPA_VERSION $(pwd) ) + - build-rpms: | + cd $WORKDIR + make -C $WORKDIR/vespa -f .copr/Makefile srpm outdir=$WORKDIR + rpmbuild --rebuild \ + --define="_topdir $WORKDIR/vespa-rpmbuild" \ + --define "debug_package %{nil}" \ + --define "_debugsource_template %{nil}" \ + --define '_cmake_extra_opts "-DDEFAULT_VESPA_CPU_ARCH_FLAGS=-msse3 -mcx16 -mtune=intel"' \ + *.src.rpm + rm -f *.src.rpm + mv $WORKDIR/vespa-rpmbuild/RPMS/x86_64/*.rpm . + - build-container-image: | + cat <<EOF > Dockerfile + ARG VESPA_VERSION + FROM docker.io/vespaengine/vespa:\$VESPA_VERSION + USER root + RUN --mount=type=bind,target=/rpms/,source=. dnf reinstall -y /rpms/vespa*rpm && dnf clean all + USER vespa + EOF + docker build --progress plain --build-arg VESPA_VERSION=$VESPA_VERSION --tag docker.io/$IMAGE_NAME:$VESPA_VERSION \ + --tag docker.io/$IMAGE_NAME:latest --file Dockerfile . + - verify-container-image: | + # Trick to be able to use the documentation testing to verify the image built locally + docker tag docker.io/$IMAGE_NAME:$VESPA_VERSION vespaengine/vespa:latest + # Clone and setup doc tests + git clone -q --depth 1 https://github.com/vespa-engine/documentation + cd documentation + python3 -m pip install -qqq -r test/requirements.txt --user + echo -e "urls:\n - en/vespa-quick-start.html" > test/_quick-start.yaml + # Get the required vespa CLI + VESPA_CLI_VERSION=$(curl -fsSL https://api.github.com/repos/vespa-engine/vespa/releases/latest | grep -Po '"tag_name": "v\K.*?(?=")') && \ + curl -fsSL https://github.com/vespa-engine/vespa/releases/download/v${VESPA_CLI_VERSION}/vespa-cli_${VESPA_CLI_VERSION}_linux_amd64.tar.gz | tar -zxf - -C /opt && \ + ln -sf /opt/vespa-cli_${VESPA_CLI_VERSION}_linux_amd64/bin/vespa /usr/local/bin/ + # Run test + test/test.py -c test/_quick-start.yaml + - publish-test-image: | + if [[ -z $SD_PULL_REQUEST ]]; then + if curl -fsSL https://index.docker.io/v1/repositories/$IMAGE_NAME/tags/$VESPA_VERSION &> /dev/null; then + echo "Container image docker.io/$IMAGE_NAME:$VESPA_VERSION aldready exists." + else + OPT_STATE="$(set +o)" + set +x + docker login --username aressem --password "$DOCKER_HUB_DEPLOY_KEY" + eval "$OPT_STATE" + docker push docker.io/$IMAGE_NAME:$VESPA_VERSION + docker push docker.io/$IMAGE_NAME:latest + fi + fi + publish-cli-release: image: homebrew/brew:latest annotations: diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index d6c353e46c9..131460b0384 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos fnet vespalog vespalib diff --git a/searchcore/src/apps/proton/proton.cpp b/searchcore/src/apps/proton/proton.cpp index 2c47b5162ca..281d7585274 100644 --- a/searchcore/src/apps/proton/proton.cpp +++ b/searchcore/src/apps/proton/proton.cpp @@ -10,7 +10,6 @@ #include <vespa/config/common/exceptions.h> #include <vespa/config/common/configcontext.h> #include <vespa/fnet/transport.h> -#include <vespa/fastos/thread.h> #include <vespa/fastos/file.h> #include <filesystem> #include <iostream> @@ -45,7 +44,7 @@ private: static void setupSignals(); static void setup_fadvise(); Params parseParams(int argc, char **argv); - void startAndRun(FastOS_ThreadPool & threadPool, FNET_Transport & transport, int argc, char **argv); + void startAndRun(FNET_Transport & transport, int argc, char **argv); public: int main(int argc, char **argv); }; @@ -206,10 +205,10 @@ buildTransportConfig() { class Transport { public: - Transport(const fnet::TransportConfig & config, FastOS_ThreadPool & threadPool) + Transport(const fnet::TransportConfig & config) : _transport(config) { - _transport.Start(&threadPool); + _transport.Start(); } ~Transport() { _transport.ShutDown(true); @@ -222,7 +221,7 @@ private: } void -App::startAndRun(FastOS_ThreadPool & threadPool, FNET_Transport & transport, int argc, char **argv) { +App::startAndRun(FNET_Transport & transport, int argc, char **argv) { Params params = parseParams(argc, argv); LOG(debug, "identity: '%s'", params.identity.c_str()); LOG(debug, "serviceidentity: '%s'", params.serviceidentity.c_str()); @@ -231,7 +230,7 @@ App::startAndRun(FastOS_ThreadPool & threadPool, FNET_Transport & transport, int config::ConfigServerSpec configServerSpec(transport); config::ConfigUri identityUri(params.identity, std::make_shared<config::ConfigContext>(configServerSpec)); - auto protonUP = std::make_unique<proton::Proton>(threadPool, transport, identityUri, + auto protonUP = std::make_unique<proton::Proton>(transport, identityUri, (argc > 0) ? argv[0] : "proton", subscribeTimeout); proton::Proton & proton = *protonUP; proton::BootstrapConfig::SP configSnapshot = proton.init(); @@ -283,9 +282,8 @@ App::main(int argc, char **argv) try { setupSignals(); setup_fadvise(); - FastOS_ThreadPool threadPool; - Transport transport(buildTransportConfig(), threadPool); - startAndRun(threadPool, transport.transport(), argc, argv); + Transport transport(buildTransportConfig()); + startAndRun(transport.transport(), argc, argv); } catch (const vespalib::InvalidCommandLineArgumentsException &e) { LOG(warning, "Invalid commandline arguments: '%s'", e.what()); return 1; diff --git a/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp b/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp index 8eb56d218fb..c88e4a283b5 100644 --- a/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp +++ b/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp @@ -72,11 +72,11 @@ public: void finiRPC() { if (_req != nullptr) { - _req->SubRef(); + _req->internal_subref(); _req = nullptr; } if (_target != nullptr) { - _target->SubRef(); + _target->internal_subref(); _target = nullptr; } if (_frt) { diff --git a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp index 76fc875b209..6a1f71ce872 100644 --- a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp +++ b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp @@ -19,7 +19,6 @@ #include <vespa/vespalib/util/signalhandler.h> #include <iostream> #include <thread> -#include <vespa/fastos/thread.h> #include <vespa/log/log.h> LOG_SETUP("vespa-transactionlog-inspect"); @@ -37,7 +36,7 @@ using IReplayPacketHandlerUP = std::unique_ptr<IReplayPacketHandler>; struct DummyFileHeaderContext : public FileHeaderContext { using UP = std::unique_ptr<DummyFileHeaderContext>; - virtual void addTags(vespalib::GenericHeader &, const vespalib::string &) const override {} + void addTags(vespalib::GenericHeader &, const vespalib::string &) const override {} }; @@ -48,29 +47,18 @@ class ConfigFile using SP = std::shared_ptr<ConfigFile>; vespalib::string _name; - time_t _modTime; std::vector<char> _content; public: ConfigFile(); - - const vespalib::string & - getName() const - { - return _name; - } - - vespalib::nbostream & - deserialize(vespalib::nbostream &stream); - - void - print() const; + const vespalib::string & getName() const { return _name; } + vespalib::nbostream & deserialize(vespalib::nbostream &stream); + void print() const; }; ConfigFile::ConfigFile() : _name(), - _modTime(0), _content() { } @@ -80,10 +68,9 @@ vespalib::nbostream & ConfigFile::deserialize(vespalib::nbostream &stream) { stream >> _name; - assert(strchr(_name.c_str(), '/') == NULL); + assert(strchr(_name.c_str(), '/') == nullptr); int64_t modTime; stream >> modTime; - _modTime = modTime; uint32_t sz; stream >> sz; _content.resize(sz); @@ -96,9 +83,8 @@ ConfigFile::deserialize(vespalib::nbostream &stream) void ConfigFile::print() const { - std::cout << "Name: " << _name << "\n" << - "ModTime: " << _modTime << "\n" << - "Content-Length: " << _content.size() << "\n\n"; + std::cout << "Name: " << _name << "\n" + << "Content-Length: " << _content.size() << "\n\n"; std::cout.write(&_content[0], _content.size()); std::cout << "\n-----------------------------" << std::endl; } @@ -121,12 +107,9 @@ struct DummyStreamHandler : public NewConfigOperation::IStreamHandler { { } - virtual void - serializeConfig(SerialNum, vespalib::nbostream &) override - { - } + void serializeConfig(SerialNum, vespalib::nbostream &) override { } - virtual void + void deserializeConfig(SerialNum, vespalib::nbostream &is) override { _cfs.clear(); @@ -154,7 +137,7 @@ DocTypeRepo::DocTypeRepo(const std::string &configDir) { } -DocTypeRepo::~DocTypeRepo() {} +DocTypeRepo::~DocTypeRepo() = default; /** @@ -187,10 +170,8 @@ public: void replay(const NewConfigOperation &op) override { print(op); - using I = std::map<std::string, ConfigFile>::const_iterator; - for (I i(_streamHandler._cfs.begin()), ie(_streamHandler._cfs.end()); - i != ie; ++i) { - i->second.print(); + for (const auto & entry : _streamHandler._cfs) { + entry.second.print(); } } @@ -369,7 +350,7 @@ BaseOptions::BaseOptions(int argc, const char* const* argv) _opts.addOption("tlsname", tlsName, std::string("tls"), "Name of the tls"); _opts.addOption("listenport", listenPort, 13701, "Tcp listen port"); } -BaseOptions::~BaseOptions() {} +BaseOptions::~BaseOptions() = default; /** * Base class for a utility with tls server and tls client. @@ -379,7 +360,6 @@ class BaseUtility : public Utility protected: const BaseOptions &_bopts; DummyFileHeaderContext _fileHeader; - FastOS_ThreadPool _threadPool; FNET_Transport _transport; TransLogServer _server; client::TransLogClient _client; @@ -388,17 +368,16 @@ public: BaseUtility(const BaseOptions &bopts) : _bopts(bopts), _fileHeader(), - _threadPool(), _transport(), _server(_transport, _bopts.tlsName, _bopts.listenPort, _bopts.tlsDir, _fileHeader), _client(_transport, vespalib::make_string("tcp/localhost:%d", _bopts.listenPort)) { - _transport.Start(&_threadPool); + _transport.Start(); } ~BaseUtility() override { _transport.ShutDown(true); } - virtual int run() override = 0; + int run() override = 0; }; @@ -413,7 +392,7 @@ struct ListDomainsOptions : public BaseOptions _opts.setSyntaxMessage("Utility to list all domains in a tls"); } static std::string command() { return "listdomains"; } - virtual Utility::UP createUtility() const override; + Utility::UP createUtility() const override; }; /** @@ -426,7 +405,7 @@ public: : BaseUtility(opts) { } - virtual int run() override { + int run() override { std::cout << ListDomainsOptions::command() << ": " << _bopts.toString() << std::endl; std::vector<vespalib::string> domains; @@ -448,7 +427,7 @@ public: Utility::UP ListDomainsOptions::createUtility() const { - return Utility::UP(new ListDomainsUtility(*this)); + return std::make_unique<ListDomainsUtility>(*this); } @@ -482,7 +461,7 @@ DumpOperationsOptions::DumpOperationsOptions(int argc, const char* const* argv) _opts.addOption("configdir", configDir, "Config directory (with documenttypes.cfg)"); _opts.setSyntaxMessage("Utility to dump a range of operations ([first,last]) in a tls domain"); } -DumpOperationsOptions::~DumpOperationsOptions() {} +DumpOperationsOptions::~DumpOperationsOptions() = default; /** @@ -494,7 +473,7 @@ protected: const DumpOperationsOptions &_oopts; virtual IReplayPacketHandlerUP createHandler(DocumentTypeRepo &repo) { - return IReplayPacketHandlerUP(new OperationPrinter(repo)); + return std::make_unique<OperationPrinter>(repo); } int doRun() { @@ -520,7 +499,7 @@ public: _oopts(oopts) { } - virtual int run() override { + int run() override { std::cout << DumpOperationsOptions::command() << ": " << _oopts.toString() << std::endl; return doRun(); } @@ -529,7 +508,7 @@ public: Utility::UP DumpOperationsOptions::createUtility() const { - return Utility::UP(new DumpOperationsUtility(*this)); + return std::make_unique<DumpOperationsUtility>(*this); } @@ -543,18 +522,18 @@ struct DumpDocumentsOptions : public DumpOperationsOptions DumpDocumentsOptions(int argc, const char* const* argv); ~DumpDocumentsOptions(); static std::string command() { return "dumpdocuments"; } - virtual void parse() override { + void parse() override { DumpOperationsOptions::parse(); if (format != "xml" && format != "text") { throw vespalib::InvalidCommandLineArgumentsException("Expected 'format' to be 'xml' or 'text'"); } } - virtual std::string toString() const override { + std::string toString() const override { return vespalib::make_string("%s, format=%s, verbose=%s", DumpOperationsOptions::toString().c_str(), format.c_str(), (verbose ? "true" : "false")); } - virtual Utility::UP createUtility() const override; + Utility::UP createUtility() const override; }; DumpDocumentsOptions::DumpDocumentsOptions(int argc, const char* const* argv) @@ -574,7 +553,7 @@ class DumpDocumentsUtility : public DumpOperationsUtility { protected: const DumpDocumentsOptions &_dopts; - virtual IReplayPacketHandlerUP createHandler(DocumentTypeRepo &repo) override { + IReplayPacketHandlerUP createHandler(DocumentTypeRepo &repo) override { return IReplayPacketHandlerUP(new DocumentPrinter(repo, _dopts.format == "xml", _dopts.verbose)); } @@ -584,7 +563,7 @@ public: _dopts(dopts) { } - virtual int run() override { + int run() override { std::cout << DumpDocumentsOptions::command() << ": " << _oopts.toString() << std::endl; return doRun(); } @@ -593,7 +572,7 @@ public: Utility::UP DumpDocumentsOptions::createUtility() const { - return Utility::UP(new DumpDocumentsUtility(*this)); + return std::make_unique<DumpDocumentsUtility>(*this); } @@ -633,8 +612,8 @@ public: int main(int argc, char **argv); }; -App::App() {} -App::~App() {} +App::App() = default; +App::~App() = default; int App::main(int argc, char **argv) { @@ -654,7 +633,7 @@ App::main(int argc, char **argv) { combineFirstArgs(argv); opts.reset(new DumpDocumentsOptions(argc-1, argv+1)); } - if (opts.get() != NULL) { + if (opts) { try { opts->parse(); } catch (const vespalib::InvalidCommandLineArgumentsException &e) { diff --git a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp index b94df75c7be..dfcc8d1f08b 100644 --- a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp +++ b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp @@ -14,7 +14,6 @@ #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/test/directory_handler.h> #include <vespa/searchcommon/attribute/config.h> -#include <vespa/vespalib/datastore/datastorebase.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/foreground_thread_executor.h> #include <vespa/vespalib/util/foregroundtaskexecutor.h> @@ -23,6 +22,7 @@ #include <vespa/vespalib/util/threadstackexecutor.h> #include <filesystem> #include <thread> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP("attributeflush_test"); @@ -529,20 +529,20 @@ Test::requireThatLastFlushTimeIsReported() EXPECT_EQUAL(vespalib::system_time(), ft->getLastFlushTime()); ft->initFlush(200, std::make_shared<search::FlushToken>())->run(); EXPECT_TRUE(FastOS_File::Stat("flush/a9/snapshot-200", &stat)); - EXPECT_EQUAL(seconds(stat._modifiedTime), duration_cast<seconds>(ft->getLastFlushTime().time_since_epoch())); + EXPECT_EQUAL(stat._modifiedTime, ft->getLastFlushTime()); } { // snapshot flushed AttributeManagerFixture amf(f); AttributeManager &am = amf._m; amf.addAttribute("a9"); IFlushTarget::SP ft = am.getFlushable("a9"); - EXPECT_EQUAL(seconds(stat._modifiedTime), duration_cast<seconds>(ft->getLastFlushTime().time_since_epoch())); + EXPECT_EQUAL(stat._modifiedTime, ft->getLastFlushTime()); { // updated flush time after nothing to flush std::this_thread::sleep_for(1100ms); std::chrono::seconds now = duration_cast<seconds>(vespalib::system_clock::now().time_since_epoch()); Executor::Task::UP task = ft->initFlush(200, std::make_shared<search::FlushToken>()); EXPECT_FALSE(task); - EXPECT_LESS(seconds(stat._modifiedTime), ft->getLastFlushTime().time_since_epoch()); + EXPECT_LESS(stat._modifiedTime, ft->getLastFlushTime()); EXPECT_APPROX(now.count(), duration_cast<seconds>(ft->getLastFlushTime().time_since_epoch()).count(), 3); } } diff --git a/searchcore/src/tests/proton/common/timer/timer_test.cpp b/searchcore/src/tests/proton/common/timer/timer_test.cpp index ac82767cd7c..4ff970df84e 100644 --- a/searchcore/src/tests/proton/common/timer/timer_test.cpp +++ b/searchcore/src/tests/proton/common/timer/timer_test.cpp @@ -1,6 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/thread.h> #include <vespa/fnet/transport.h> #include <vespa/searchcore/proton/common/scheduled_forward_executor.h> #include <vespa/searchcore/proton/common/scheduledexecutor.h> @@ -46,17 +45,15 @@ make_scheduled_executor<ScheduledForwardExecutor>(FNET_Transport& transport, ves template <typename ScheduledT> class ScheduledExecutorTest : public testing::Test { public: - FastOS_ThreadPool threadPool; FNET_Transport transport; vespalib::ThreadStackExecutor executor; std::unique_ptr<ScheduledT> timer; ScheduledExecutorTest() - : threadPool(), - transport(), + : transport(), executor(1) { - transport.Start(&threadPool); + transport.Start(); timer = make_scheduled_executor<ScheduledT>(transport, executor); } ~ScheduledExecutorTest() { diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index 0234d5fbedf..5afe0a0c5a2 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -4,7 +4,6 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/test/make_bucket_space.h> -#include <vespa/fastos/thread.h> #include <vespa/persistence/dummyimpl/dummy_bucket_executor.h> #include <vespa/searchcore/proton/attribute/attribute_config_inspector.h> #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> @@ -82,9 +81,9 @@ namespace { VESPA_THREAD_STACK_TAG(my_executor_init); void -sampleThreadId(FastOS_ThreadId *threadId) +sampleThreadId(std::thread::id *threadId) { - *threadId = FastOS_Thread::GetCurrentThreadId(); + *threadId = std::this_thread::get_id(); } } // namespace @@ -207,12 +206,12 @@ class MyFeedHandler : public IDocumentMoveHandler, public IHeartBeatHandler, public IOperationStorer { - FastOS_ThreadId _executorThreadId; + std::thread::id _executorThreadId; std::vector<MyDocumentSubDB *> _subDBs; SerialNum _serialNum; std::atomic<uint32_t> _heartBeats; public: - explicit MyFeedHandler(FastOS_ThreadId &executorThreadId); + explicit MyFeedHandler(std::thread::id executorThreadId); ~MyFeedHandler() override; @@ -241,7 +240,7 @@ public: class MyExecutor : public vespalib::ThreadStackExecutorBase { public: - FastOS_ThreadId _threadId; + std::thread::id _threadId; MyExecutor(); bool acceptNewTask(unique_lock &, std::condition_variable &) override { @@ -604,7 +603,7 @@ MyDocumentSubDB::getNumUsedLids() const } -MyFeedHandler::MyFeedHandler(FastOS_ThreadId &executorThreadId) +MyFeedHandler::MyFeedHandler(std::thread::id executorThreadId) : IDocumentMoveHandler(), IPruneRemovedDocumentsHandler(), IHeartBeatHandler(), @@ -622,8 +621,7 @@ MyFeedHandler::~MyFeedHandler() = default; bool MyFeedHandler::isExecutorThread() const { - FastOS_ThreadId threadId(FastOS_Thread::GetCurrentThreadId()); - return FastOS_Thread::CompareThreadIds(_executorThreadId, threadId); + return (_executorThreadId == std::this_thread::get_id()); } diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index 19035eaa7f7..880bf8aa3e0 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -4,7 +4,6 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/repo/configbuilder.h> -#include <vespa/document/fieldvalue/document.h> #include <vespa/searchcore/proton/test/transport_helper.h> #include <vespa/searchcorespi/index/index_manager_stats.h> #include <vespa/searchcorespi/index/indexcollection.h> @@ -305,7 +304,7 @@ TEST_F(IndexManagerTest, require_that_memory_index_is_flushed) runAsMaster([&]() { flushTask = target.initFlush(1, std::make_shared<search::FlushToken>()); }); flushTask->run(); EXPECT_TRUE(FastOS_File::Stat("test_data/index.flush.1", &stat)); - EXPECT_EQ(seconds(stat._modifiedTime), duration_cast<seconds>(target.getLastFlushTime().time_since_epoch())); + EXPECT_EQ(stat._modifiedTime, target.getLastFlushTime()); sources = get_source_collection(); EXPECT_EQ(2u, sources->getSourceCount()); @@ -323,7 +322,7 @@ TEST_F(IndexManagerTest, require_that_memory_index_is_flushed) { // verify last flush time when loading disk index resetIndexManager(); IndexFlushTarget target(_index_manager->getMaintainer()); - EXPECT_EQ(seconds(stat._modifiedTime), duration_cast<seconds>(target.getLastFlushTime().time_since_epoch())); + EXPECT_EQ(stat._modifiedTime, target.getLastFlushTime()); // updated serial number & flush time when nothing to flush std::this_thread::sleep_for(2s); @@ -332,7 +331,7 @@ TEST_F(IndexManagerTest, require_that_memory_index_is_flushed) runAsMaster([&]() { task = target.initFlush(2, std::make_shared<search::FlushToken>()); }); EXPECT_FALSE(task); EXPECT_EQ(2u, target.getFlushedSerialNum()); - EXPECT_LT(seconds(stat._modifiedTime), duration_cast<seconds>(target.getLastFlushTime().time_since_epoch())); + EXPECT_LT(stat._modifiedTime, target.getLastFlushTime()); EXPECT_NEAR(now.count(), duration_cast<seconds>(target.getLastFlushTime().time_since_epoch()).count(), 2); } } diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index afd224c55de..13371521718 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -305,7 +305,7 @@ TEST_FFF("require that proton config fetcher follows changes to bootstrap", ConfigTestFixture("search"), ProtonConfigOwner(), ProtonConfigFetcher(f1.transport.transport(), ConfigUri(f1.configId, f1.context), f2, 60s)) { - f3.start(f1.transport.threadPool()); + f3.start(); ASSERT_TRUE(f2._configured); ASSERT_TRUE(f1.configEqual(f2.getBootstrapConfig())); f2._configured = false; @@ -320,7 +320,7 @@ TEST_FFF("require that proton config fetcher follows changes to doctypes", ConfigTestFixture("search"), ProtonConfigOwner(), ProtonConfigFetcher(f1.transport.transport(), ConfigUri(f1.configId, f1.context), f2, 60s)) { - f3.start(f1.transport.threadPool()); + f3.start(); f2._configured = false; f1.addDocType("typea"); @@ -340,7 +340,7 @@ TEST_FFF("require that proton config fetcher reconfigures dbowners", ConfigTestFixture("search"), ProtonConfigOwner(), ProtonConfigFetcher(f1.transport.transport(), ConfigUri(f1.configId, f1.context), f2, 60s)) { - f3.start(f1.transport.threadPool()); + f3.start(); ASSERT_FALSE(f2.getDocumentDBConfig("typea")); // Add db and verify that config for db is provided diff --git a/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_controller.cpp b/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_controller.cpp index 606188f72c2..4c13ff7d762 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_controller.cpp +++ b/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_controller.cpp @@ -56,7 +56,7 @@ BmClusterController::propagate_cluster_state(uint32_t node_idx, bool distributor auto target = target_resolver->resolve_rpc_target(storage_address, fake_bucket_id); target->get()->InvokeSync(req, 10.0); // 10 seconds timeout assert(!req->IsError()); - req->SubRef(); + req->internal_subref(); } void diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp index 16be2c1bd25..d39d2873edb 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp @@ -14,6 +14,7 @@ #include <vespa/searchlib/attribute/attribute_header.h> #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/fastos/file.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".proton.attribute.attribute_initializer"); diff --git a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp index 7d9b1b3b06a..e412dabc7c1 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp @@ -83,6 +83,7 @@ getValue(const Context &context) const case BasicType::PREDICATE: case BasicType::TENSOR: case BasicType::REFERENCE: + case BasicType::RAW: throw IllegalArgumentException(make_string("Attribute '%s' of type '%s' can not be used for selection", v.getName().c_str(), BasicType(v.getBasicType()).asString())); case BasicType::MAX_TYPE: diff --git a/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp b/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp index c51771df265..2235f16ae94 100644 --- a/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp @@ -65,7 +65,7 @@ AttrVisitor::AttrVisitor(const search::IAttributeManager &amgr, CachedSelect::At AttrVisitor::~AttrVisitor() = default; bool isSingleValueThatWeHandle(BasicType type) { - return (type != BasicType::PREDICATE) && (type != BasicType::TENSOR) && (type != BasicType::REFERENCE); + return (type != BasicType::PREDICATE) && (type != BasicType::TENSOR) && (type != BasicType::REFERENCE) && (type != BasicType::RAW); } void diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp index 4b83b7d5af9..3e577bf6cbe 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp @@ -10,7 +10,6 @@ #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/searchlib/common/flush_token.h> #include <vespa/vespalib/util/cpu_usage.h> -#include <thread> #include <vespa/log/log.h> LOG_SETUP(".proton.flushengine.flushengine"); @@ -86,7 +85,8 @@ FlushEngine::FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory> tlsStats _maxConcurrent(numThreads), _idleInterval(idleInterval), _taskId(0), - _threadPool(), + _thread(), + _has_thread(false), _strategy(std::move(strategy)), _priorityStrategy(), _executor(numThreads, CpuUsage::wrap(flush_engine_executor, CpuUsage::Category::COMPACT)), @@ -111,9 +111,7 @@ FlushEngine::~FlushEngine() FlushEngine & FlushEngine::start() { - if (_threadPool.NewThread(this) == nullptr) { - throw vespalib::IllegalStateException("Failed to start engine thread."); - } + _thread = std::thread([this](){run();}); return *this; } @@ -127,7 +125,9 @@ FlushEngine::close() _closed = true; _cond.notify_all(); } - _threadPool.Close(); + if (_thread.joinable()) { + _thread.join(); + } _executor.shutdown(); _executor.sync(); return *this; @@ -168,8 +168,9 @@ FlushEngine::wait(vespalib::duration minimumWaitTimeIfReady, bool ignorePendingP } void -FlushEngine::Run(FastOS_ThreadInterface *, void *) +FlushEngine::run() { + _has_thread = true; bool shouldIdle = false; vespalib::string prevFlushName; while (wait(shouldIdle ? _idleInterval : vespalib::duration::zero(), false)) { @@ -190,6 +191,7 @@ FlushEngine::Run(FastOS_ThreadInterface *, void *) } _executor.sync(); prune(); + _has_thread = false; } namespace { diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h index 4b15c3503f3..1d6ed763ff6 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h @@ -7,7 +7,7 @@ #include <vespa/searchcore/proton/common/doctypename.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/time.h> -#include <vespa/fastos/thread.h> +#include <thread> #include <set> #include <mutex> #include <condition_variable> @@ -18,7 +18,7 @@ namespace proton { namespace flushengine { class ITlsStatsFactory; } -class FlushEngine final : public FastOS_Runnable +class FlushEngine { public: class FlushMeta { @@ -54,7 +54,8 @@ private: const uint32_t _maxConcurrent; const vespalib::duration _idleInterval; uint32_t _taskId; - FastOS_ThreadPool _threadPool; + std::thread _thread; + std::atomic<bool> _has_thread; IFlushStrategy::SP _strategy; mutable IFlushStrategy::SP _priorityStrategy; vespalib::ThreadStackExecutor _executor; @@ -109,7 +110,7 @@ public: /** * Destructor. Waits for all pending tasks to complete. */ - ~FlushEngine() override; + ~FlushEngine(); /** * Observe and reset internal executor stats @@ -129,7 +130,8 @@ public: * @return This, to allow chaining. */ FlushEngine &start(); - + bool has_thread() const { return _has_thread; } + /** * Stops the scheduling thread and. This will prevent any more flush * requests being performed on the attached handlers, allowing you to flush @@ -168,7 +170,7 @@ public: */ IFlushHandler::SP removeFlushHandler(const DocTypeName &docTypeName); - void Run(FastOS_ThreadInterface *thread, void *arg) override; + void run(); FlushMetaSet getCurrentlyFlushingSet() const; diff --git a/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp b/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp index 3512d2eebad..2c26b043fad 100644 --- a/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp @@ -2,6 +2,7 @@ #include "index_writer.h" #include <vespa/document/fieldvalue/document.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".proton.server.indexadapter"); diff --git a/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h b/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h index 177d19b06f9..cd4d1686eeb 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h +++ b/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h @@ -3,13 +3,14 @@ #pragma once #include <vespa/searchlib/queryeval/begin_and_end_id.h> -#include <vespa/fastos/types.h> #include <mutex> #include <condition_variable> #include <atomic> #include <algorithm> #include <vector> +#define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden"))) + namespace proton::matching { /** diff --git a/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.cpp index c2d09fa341b..9e9d51abd57 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.cpp @@ -29,6 +29,7 @@ ContentProtonMetrics::ProtonExecutorMetrics::~ProtonExecutorMetrics() = default; ContentProtonMetrics::ContentProtonMetrics() : metrics::MetricSet("content.proton", {}, "Search engine metrics", nullptr), + configGeneration("config.generation", {}, "The oldest config generation used by this process", this), transactionLog(this), resourceUsage(this), executor(this), diff --git a/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.h index 127e32ada07..c82a6804380 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.h @@ -41,6 +41,7 @@ struct ContentProtonMetrics : metrics::MetricSet ~SessionCacheMetrics() override; }; + metrics::LongValueMetric configGeneration; TransLogServerMetrics transactionLog; ResourceUsageMetrics resourceUsage; ProtonExecutorMetrics executor; diff --git a/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp b/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp index 1ccb3956fc2..4f7e0e66d9f 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp @@ -42,7 +42,7 @@ MetricsEngine::start(const config::ConfigUri &) void MetricsEngine::addMetricsHook(metrics::UpdateHook &hook) { - _manager->addMetricUpdateHook(hook, 5); + _manager->addMetricUpdateHook(hook); } void diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 9a84383c2f4..3f28f75c521 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -94,7 +94,7 @@ class MetricsUpdateHook : public metrics::UpdateHook { DocumentDB &_db; public: explicit MetricsUpdateHook(DocumentDB &s) - : metrics::UpdateHook("documentdb-hook"), + : metrics::UpdateHook("documentdb-hook", 5s), _db(s) {} void updateMetrics(const MetricLockGuard & guard) override { diff --git a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp index 74f6a622661..27650b27c77 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp @@ -3,7 +3,6 @@ #include "executor_thread_service.h" #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/gate.h> -#include <vespa/fastos/thread.h> using vespalib::makeLambdaTask; using vespalib::Executor; @@ -14,27 +13,20 @@ using vespalib::SyncableThreadExecutor; namespace proton { -namespace internal { - -struct ThreadId { - FastOS_ThreadId _id; -}; -} - namespace { void -sampleThreadId(FastOS_ThreadId *threadId) +sampleThreadId(std::thread::id *threadId) { - *threadId = FastOS_Thread::GetCurrentThreadId(); + *threadId = std::this_thread::get_id(); } -std::unique_ptr<internal::ThreadId> +std::thread::id getThreadId(ThreadExecutor &executor) { - std::unique_ptr<internal::ThreadId> id = std::make_unique<internal::ThreadId>(); + std::thread::id id; vespalib::Gate gate; - executor.execute(makeLambdaTask([threadId=&id->_id, &gate] { + executor.execute(makeLambdaTask([threadId = &id, &gate] { sampleThreadId(threadId); gate.countDown(); })); @@ -74,8 +66,7 @@ ExecutorThreadService::run(Runnable &runnable) bool ExecutorThreadService::isCurrentThread() const { - FastOS_ThreadId currentThreadId = FastOS_Thread::GetCurrentThreadId(); - return FastOS_Thread::CompareThreadIds(_threadId->_id, currentThreadId); + return (_threadId == std::this_thread::get_id()); } vespalib::ExecutorStats ExecutorThreadService::getStats() { @@ -118,8 +109,7 @@ SyncableExecutorThreadService::run(Runnable &runnable) bool SyncableExecutorThreadService::isCurrentThread() const { - FastOS_ThreadId currentThreadId = FastOS_Thread::GetCurrentThreadId(); - return FastOS_Thread::CompareThreadIds(_threadId->_id, currentThreadId); + return (_threadId == std::this_thread::get_id()); } vespalib::ExecutorStats diff --git a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h index 7298b81611a..48b8d8be9b9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h +++ b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h @@ -3,10 +3,10 @@ #include <vespa/searchcorespi/index/i_thread_service.h> #include <vespa/vespalib/util/threadexecutor.h> +#include <thread> namespace proton { -namespace internal { struct ThreadId; } /** * Implementation of IThreadService using an underlying thread stack executor * with a single thread. @@ -15,7 +15,7 @@ class ExecutorThreadService : public searchcorespi::index::IThreadService { private: vespalib::ThreadExecutor &_executor; - std::unique_ptr<internal::ThreadId> _threadId; + std::thread::id _threadId; public: ExecutorThreadService(vespalib::ThreadExecutor &executor); @@ -39,8 +39,8 @@ public: class SyncableExecutorThreadService : public searchcorespi::index::ISyncableThreadService { private: - vespalib::SyncableThreadExecutor &_executor; - std::unique_ptr<internal::ThreadId> _threadId; + vespalib::SyncableThreadExecutor &_executor; + std::thread::id _threadId; public: SyncableExecutorThreadService(vespalib::SyncableThreadExecutor &executor); @@ -66,5 +66,3 @@ public: }; } // namespace proton - - diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp index f22f57979b4..e4a4e6761aa 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp @@ -21,6 +21,7 @@ #include <filesystem> #include <sstream> #include <cassert> +#include <cinttypes> #include <fcntl.h> #include <vespa/log/log.h> @@ -94,9 +95,8 @@ class ConfigFile { using SP = std::shared_ptr<ConfigFile>; - vespalib::string _name; - time_t _modTime; - std::vector<char> _content; + vespalib::string _name; + std::vector<char> _content; public: ConfigFile(); @@ -111,7 +111,6 @@ public: ConfigFile::ConfigFile() : _name(), - _modTime(0), _content() { } @@ -120,7 +119,6 @@ ConfigFile::~ConfigFile() = default; ConfigFile::ConfigFile(const vespalib::string &name, const vespalib::string &fullName) : _name(name), - _modTime(0), _content() { FastOS_File file; @@ -130,7 +128,6 @@ ConfigFile::ConfigFile(const vespalib::string &name, const vespalib::string &ful int64_t fileSize = file.getSize(); _content.resize(fileSize); file.ReadBuf(_content.data(), fileSize); - _modTime = file.GetModificationTime(); } nbostream & @@ -138,7 +135,7 @@ ConfigFile::serialize(nbostream &stream) const { assert(strchr(_name.c_str(), '/') == nullptr); stream << _name; - stream << static_cast<int64_t>(_modTime);; + stream << int64_t(0ul); // Used to be modtime => unused uint32_t sz = _content.size(); stream << sz; stream.write(_content.data(), sz); @@ -150,9 +147,8 @@ ConfigFile::deserialize(nbostream &stream) { stream >> _name; assert(strchr(_name.c_str(), '/') == nullptr); - int64_t modTime; - stream >> modTime; - _modTime = modTime; + int64_t unused_modTime; + stream >> unused_modTime; uint32_t sz; stream >> sz; _content.resize(sz); diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp index c86019005dc..fa8b86416d0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp @@ -6,8 +6,7 @@ #include <vespa/searchcorespi/index/i_thread_service.h> #include <vespa/searchcore/proton/common/scheduledexecutor.h> #include <vespa/vespalib/util/lambdatask.h> -#include <vespa/fastos/thread.h> -#include <thread> +#include <vespa/vespalib/util/thread.h> #include <vespa/log/log.h> LOG_SETUP(".proton.server.maintenancecontroller"); @@ -16,6 +15,7 @@ using document::BucketId; using vespalib::Executor; using vespalib::MonitoredRefCount; using vespalib::makeLambdaTask; +using vespalib::thread::as_zu; namespace proton { @@ -69,7 +69,7 @@ MaintenanceController::killJobs() } // Called by master write thread assert(_masterThread.isCurrentThread()); - LOG(debug, "killJobs(): threadId=%zu", (size_t)FastOS_Thread::GetCurrentThreadId()); + LOG(debug, "killJobs(): threadId=%zu", as_zu(std::this_thread::get_id())); _periodicTaskHandles.clear(); // No need to take _jobsLock as modification of _jobs also happens in master write thread. for (auto &job : _jobs) { @@ -99,7 +99,7 @@ void MaintenanceController::performHoldJobs(JobList jobs) { // Called by master write thread - LOG(debug, "performHoldJobs(): threadId=%zu", (size_t)FastOS_Thread::GetCurrentThreadId()); + LOG(debug, "performHoldJobs(): threadId=%zu", as_zu(std::this_thread::get_id())); (void) jobs; } diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp index 5e6cee94292..e580066ec17 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "maintenancejobrunner.h" -#include <vespa/fastos/thread.h> #include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/lambdatask.h> @@ -54,10 +53,9 @@ MaintenanceJobRunner::runJobInExecutor() } bool finished = _job->run(); if (LOG_WOULD_LOG(debug)) { - FastOS_ThreadId threadId = FastOS_Thread::GetCurrentThreadId(); LOG(debug, - "runJobInExecutor(): job='%s', task='%p', threadId=%" PRIu64 "", - _job->getName().c_str(), this, (uint64_t)threadId); + "runJobInExecutor(): job='%s', task='%p'", + _job->getName().c_str(), this); } if (!finished) { addExecutorTask(); diff --git a/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp b/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp index 50a499c8a73..3222cbc3a06 100644 --- a/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp @@ -2,6 +2,7 @@ #include "memory_flush_config_updater.h" #include <vespa/vespalib/util/size_literals.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".proton.server.memory_flush_config_updater"); diff --git a/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp b/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp index f8d7519fd0c..fbc6e2beaf5 100644 --- a/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/time.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".proton.server.memoryflush"); diff --git a/searchcore/src/vespa/searchcore/proton/server/prepare_restart_handler.cpp b/searchcore/src/vespa/searchcore/proton/server/prepare_restart_handler.cpp index 97b66600817..0ec0e6a1147 100644 --- a/searchcore/src/vespa/searchcore/proton/server/prepare_restart_handler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/prepare_restart_handler.cpp @@ -21,7 +21,7 @@ bool PrepareRestartHandler::prepareRestart(const ProtonConfig &protonCfg) { std::unique_lock lock(_mutex); - if (!_flushEngine.HasThread()) { + if (!_flushEngine.has_thread()) { return false; } if (!_running) { diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 54009ef60f4..8ec904760cf 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -136,7 +136,7 @@ struct MetricsUpdateHook : metrics::UpdateHook { Proton &self; explicit MetricsUpdateHook(Proton &s) - : metrics::UpdateHook("proton-hook"), + : metrics::UpdateHook("proton-hook", 5s), self(s) {} void updateMetrics(const MetricLockGuard &guard) override { @@ -215,7 +215,7 @@ Proton::ProtonFileHeaderContext::setClusterName(const vespalib::string & cluster } -Proton::Proton(FastOS_ThreadPool & threadPool, FNET_Transport & transport, const config::ConfigUri & configUri, +Proton::Proton(FNET_Transport & transport, const config::ConfigUri & configUri, const vespalib::string &progName, vespalib::duration subscribeTimeout) : IProtonConfigurerOwner(), search::engine::MonitorServer(), @@ -225,7 +225,6 @@ Proton::Proton(FastOS_ThreadPool & threadPool, FNET_Transport & transport, const ComponentConfigProducer(), _cpu_util(), _hw_info(), - _threadPool(threadPool), _transport(transport), _configUri(configUri), _mutex(), @@ -277,7 +276,7 @@ Proton::init() { assert( ! _initStarted && ! _initComplete ); _initStarted = true; - _protonConfigFetcher.start(_threadPool); + _protonConfigFetcher.start(); auto configSnapshot = _protonConfigurer.getPendingConfigSnapshot(); assert(configSnapshot); auto bootstrapConfig = configSnapshot->getBootstrapConfig(); @@ -757,7 +756,7 @@ Proton::ping(std::unique_ptr<MonitorRequest>, MonitorClient &) bool Proton::triggerFlush() { - if (!_flushEngine || ! _flushEngine->HasThread()) { + if (!_flushEngine || ! _flushEngine->has_thread()) { return false; } _flushEngine->triggerFlush(); @@ -796,6 +795,7 @@ Proton::updateMetrics(const metrics::MetricLockGuard &) { { ContentProtonMetrics &metrics = _metricsEngine->root(); + metrics.configGeneration.set(getConfigGeneration()); auto tls = _tls->getTransLogServer(); if (tls) { metrics.transactionLog.update(tls->getDomainStats()); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index bf84650867a..a5da72671d2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -88,7 +88,6 @@ private: vespalib::CpuUtil _cpu_util; HwInfo _hw_info; - FastOS_ThreadPool & _threadPool; FNET_Transport & _transport; const config::ConfigUri _configUri; mutable std::shared_mutex _mutex; @@ -156,7 +155,7 @@ public: using UP = std::unique_ptr<Proton>; using SP = std::shared_ptr<Proton>; - Proton(FastOS_ThreadPool & threadPool, FNET_Transport & transport, const config::ConfigUri & configUri, + Proton(FNET_Transport & transport, const config::ConfigUri & configUri, const vespalib::string &progName, vespalib::duration subscribeTimeout); ~Proton() override; @@ -193,7 +192,6 @@ public: const std::shared_ptr<DocumentDBConfig> &documentDBConfig, InitializeThreads initializeThreads); metrics::MetricManager & getMetricManager(); - FastOS_ThreadPool & getThreadPool() { return _threadPool; } bool triggerFlush(); bool prepareRestart(); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp index 64a09a0bb25..0311cf4a48b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp @@ -29,6 +29,7 @@ ProtonConfigFetcher::ProtonConfigFetcher(FNET_Transport & transport, const confi _cond(), _dbManagerMap(), _running(false), + _thread(), _oldDocumentTypeRepos(), _currentDocumentTypeRepo() { @@ -40,10 +41,8 @@ ProtonConfigFetcher::~ProtonConfigFetcher() } void -ProtonConfigFetcher::Run(FastOS_ThreadInterface * thread, void *arg) +ProtonConfigFetcher::run() { - (void) arg; - (void) thread; while (!_retriever.isClosed()) { try { fetchConfigs(); @@ -166,17 +165,13 @@ ProtonConfigFetcher::getGeneration() const } void -ProtonConfigFetcher::start(FastOS_ThreadPool & threadPool) +ProtonConfigFetcher::start() { fetchConfigs(); lock_guard guard(_mutex); if (_running) return; _running = true; - if (threadPool.NewThread(this, nullptr) == nullptr) { - _running = false; - throw vespalib::IllegalStateException( - "Failed starting thread for proton config fetcher"); - } + _thread = std::thread([this](){run();}); } void @@ -189,6 +184,9 @@ ProtonConfigFetcher::close() while (_running) { _cond.wait(guard); } + if (_thread.joinable()) { + _thread.join(); + } } void diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h index b0046e17a63..3914d016734 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h @@ -4,11 +4,11 @@ #include "bootstrapconfigmanager.h" #include "i_document_db_config_owner.h" -#include <vespa/fastos/thread.h> #include <vespa/searchcore/proton/common/doctypename.h> #include <vespa/config/retriever/configretriever.h> #include <vespa/config/subscription/configuri.h> #include <deque> +#include <thread> class FNET_Transport; @@ -24,13 +24,13 @@ class IProtonConfigurer; * A ProtonConfigFetcher monitors all config in proton and document dbs for change * and starts proton reconfiguration if config has been reloaded. */ -class ProtonConfigFetcher : public FastOS_Runnable +class ProtonConfigFetcher { public: using BootstrapConfigSP = std::shared_ptr<BootstrapConfig>; ProtonConfigFetcher(FNET_Transport & transport, const config::ConfigUri & configUri, IProtonConfigurer &owner, vespalib::duration subscribeTimeout); - ~ProtonConfigFetcher() override; + ~ProtonConfigFetcher(); /** * Get the current config generation. */ @@ -39,14 +39,14 @@ public: /** * Start config fetcher, callbacks may come from now on. */ - void start(FastOS_ThreadPool & threadPool); + void start(); /** * Shutdown config fetcher, ensuring that no more callbacks arrive */ void close(); - void Run(FastOS_ThreadInterface * thread, void *arg) override; + void run(); private: using DBManagerMap = std::map<DocTypeName, std::shared_ptr<DocumentDBConfigManager>>; @@ -63,7 +63,8 @@ private: std::condition_variable _cond; DBManagerMap _dbManagerMap; bool _running; - + std::thread _thread; + std::deque<OldDocumentTypeRepo> _oldDocumentTypeRepos; std::shared_ptr<const document::DocumentTypeRepo> _currentDocumentTypeRepo; diff --git a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp index 587da244937..8ab71637684 100644 --- a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp @@ -136,7 +136,7 @@ RPCHooksBase::open(Params & params) initRPC(); _regAPI.registerName((params.identity + "/realtimecontroller").c_str()); _orb->Listen(params.rtcPort); - _transport->Start(&_proton.getThreadPool()); + _transport->Start(); LOG(debug, "started monitoring interface"); } diff --git a/searchcore/src/vespa/searchcore/proton/server/simpleflush.cpp b/searchcore/src/vespa/searchcore/proton/server/simpleflush.cpp index a1234ccc8fc..4de426b56d1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/simpleflush.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/simpleflush.cpp @@ -2,15 +2,14 @@ #include "simpleflush.h" #include <algorithm> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".proton.server.simpleflush"); namespace proton { -SimpleFlush::SimpleFlush() -{ -} +SimpleFlush::SimpleFlush() = default; FlushContext::List SimpleFlush::getFlushTargets(const FlushContext::List& targetList, diff --git a/searchcore/src/vespa/searchcore/proton/server/summaryadapter.cpp b/searchcore/src/vespa/searchcore/proton/server/summaryadapter.cpp index 038af801b80..b4565003e9e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/summaryadapter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/summaryadapter.cpp @@ -4,6 +4,7 @@ #include <vespa/searchcore/proton/docsummary/summarymanager.h> #include <vespa/vespalib/objects/nbostream.h> #include <cassert> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".proton.server.summaryadapter"); diff --git a/searchcore/src/vespa/searchcore/proton/test/transport_helper.cpp b/searchcore/src/vespa/searchcore/proton/test/transport_helper.cpp index d0bc4dfd4d8..92e9dadf898 100644 --- a/searchcore/src/vespa/searchcore/proton/test/transport_helper.cpp +++ b/searchcore/src/vespa/searchcore/proton/test/transport_helper.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "transport_helper.h" -#include <vespa/fastos/thread.h> #include <vespa/fnet/transport.h> #include <vespa/searchcore/proton/server/executorthreadingservice.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> @@ -12,11 +11,10 @@ namespace proton { Transport::Transport() - : _threadPool(std::make_unique<FastOS_ThreadPool>()), - _transport(std::make_unique<FNET_Transport>()), + : _transport(std::make_unique<FNET_Transport>()), _clock(std::make_unique<vespalib::TestClock>()) { - _transport->Start(_threadPool.get()); + _transport->Start(); } Transport::~Transport() { diff --git a/searchcore/src/vespa/searchcore/proton/test/transport_helper.h b/searchcore/src/vespa/searchcore/proton/test/transport_helper.h index 46ca8131041..8a724009fc1 100644 --- a/searchcore/src/vespa/searchcore/proton/test/transport_helper.h +++ b/searchcore/src/vespa/searchcore/proton/test/transport_helper.h @@ -3,8 +3,6 @@ #include <vespa/searchcorespi/index/ithreadingservice.h> -class FastOS_ThreadPool; - namespace vespalib { class TestClock; } namespace proton { @@ -19,11 +17,9 @@ public: Transport(); virtual ~Transport(); FNET_Transport & transport() { return *_transport; } - FastOS_ThreadPool & threadPool() { return *_threadPool; } const vespalib::Clock & clock() const; virtual void shutdown(); private: - std::unique_ptr<FastOS_ThreadPool> _threadPool; std::unique_ptr<FNET_Transport> _transport; std::unique_ptr<vespalib::TestClock> _clock; }; diff --git a/searchcore/src/vespa/searchcorespi/index/indexflushtarget.cpp b/searchcore/src/vespa/searchcorespi/index/indexflushtarget.cpp index e72525d0aaa..53fb21bf1ed 100644 --- a/searchcore/src/vespa/searchcorespi/index/indexflushtarget.cpp +++ b/searchcore/src/vespa/searchcorespi/index/indexflushtarget.cpp @@ -2,6 +2,7 @@ #include "indexflushtarget.h" #include <vespa/vespalib/util/size_literals.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".searchcorespi.index.indexflushtarget"); diff --git a/searchcore/src/vespa/searchcorespi/index/indexfusiontarget.cpp b/searchcore/src/vespa/searchcorespi/index/indexfusiontarget.cpp index 1df6d321f99..6755976939b 100644 --- a/searchcore/src/vespa/searchcorespi/index/indexfusiontarget.cpp +++ b/searchcore/src/vespa/searchcorespi/index/indexfusiontarget.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "indexfusiontarget.h" +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".searchcorespi.index.indexfusiontarget"); diff --git a/searchcore/src/vespa/searchcorespi/index/indexwriteutilities.cpp b/searchcore/src/vespa/searchcorespi/index/indexwriteutilities.cpp index 54d2cf6e1f5..97afce79861 100644 --- a/searchcore/src/vespa/searchcorespi/index/indexwriteutilities.cpp +++ b/searchcore/src/vespa/searchcorespi/index/indexwriteutilities.cpp @@ -164,7 +164,6 @@ IndexWriteUtilities::updateDiskIndexSchema(const vespalib::string &indexDir, LOG(error, "Could not save schema to '%s'", schemaTmpName.c_str()); } - // XXX: FastOS layer violation FastOS_StatInfo statInfo; bool statres; statres = FastOS_File::Stat(schemaOrigName.c_str(), &statInfo); @@ -183,7 +182,6 @@ IndexWriteUtilities::updateDiskIndexSchema(const vespalib::string &indexDir, } vespalib::File::sync(indexDir); } - // XXX: FastOS layer violation int renameres = ::rename(schemaTmpName.c_str(), schemaName.c_str()); if (renameres != 0) { int error = errno; diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 03429b956a4..8959a2dd2e0 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib vespaeval @@ -92,6 +91,8 @@ vespa_define_module( src/tests/attribute/posting_store src/tests/attribute/postinglist src/tests/attribute/postinglistattribute + src/tests/attribute/raw_attribute + src/tests/attribute/raw_buffer_type_mapper src/tests/attribute/reference_attribute src/tests/attribute/save_target src/tests/attribute/searchable @@ -129,6 +130,7 @@ vespa_define_module( src/tests/features src/tests/features/beta src/tests/features/bm25 + src/tests/features/closest src/tests/features/constant src/tests/features/element_completeness src/tests/features/element_similarity_feature diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index 5413907e967..be0414421fe 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -364,6 +364,8 @@ "public boolean equals(java.lang.Object)", "public int hashCode()", "public java.lang.String toString()", + "public static java.lang.String wrapInRankingExpression(java.lang.String)", + "public boolean isSimpleRankingExpressionWrapper()", "public java.lang.StringBuilder toString(java.lang.StringBuilder, com.yahoo.searchlib.rankingexpression.rule.SerializationContext, java.util.Deque, com.yahoo.searchlib.rankingexpression.rule.CompositeNode)", "public int compareTo(com.yahoo.searchlib.rankingexpression.Reference)", "public static com.yahoo.searchlib.rankingexpression.Reference fromIdentifier(java.lang.String)", @@ -371,7 +373,9 @@ "public static java.util.Optional simple(java.lang.String)", "public bridge synthetic int compareTo(java.lang.Object)" ], - "fields" : [ ] + "fields" : [ + "public static final java.lang.String RANKING_EXPRESSION_WRAPPER" + ] }, "com.yahoo.searchlib.rankingexpression.evaluation.AbstractArrayContext" : { "superClass" : "com.yahoo.searchlib.rankingexpression.evaluation.Context", diff --git a/searchlib/src/apps/uniform/uniform.cpp b/searchlib/src/apps/uniform/uniform.cpp index 784c69f4647..807b8d61a9e 100644 --- a/searchlib/src/apps/uniform/uniform.cpp +++ b/searchlib/src/apps/uniform/uniform.cpp @@ -2,8 +2,7 @@ #include <vespa/vespalib/util/signalhandler.h> #include <vespa/searchlib/bitcompression/compression.h> -#include <vespa/log/log.h> - +#include <cinttypes> static uint64_t maxExpGolombVal(uint64_t kValue, uint64_t maxBits) diff --git a/searchlib/src/apps/vespa-index-inspect/vespa-index-inspect.cpp b/searchlib/src/apps/vespa-index-inspect/vespa-index-inspect.cpp index a5d950c0f31..6278b216b52 100644 --- a/searchlib/src/apps/vespa-index-inspect/vespa-index-inspect.cpp +++ b/searchlib/src/apps/vespa-index-inspect/vespa-index-inspect.cpp @@ -16,6 +16,7 @@ #include <iostream> #include <getopt.h> #include <cstdlib> +#include <cinttypes> #include <unistd.h> #include <vespa/log/log.h> @@ -256,12 +257,12 @@ ShowPostingListSubApp::getOptions(int argc, char **argv) int c; int longopt_index = 0; static struct option longopts[] = { - { "indexdir", 1, NULL, 0 }, - { "field", 1, NULL, 0 }, - { "transpose", 0, NULL, 0 }, - { "docidlimit", 1, NULL, 0 }, - { "mindocid", 1, NULL, 0 }, - { NULL, 0, NULL, 0 } + { "indexdir", 1, nullptr, 0 }, + { "field", 1, nullptr, 0 }, + { "transpose", 0, nullptr, 0 }, + { "docidlimit", 1, nullptr, 0 }, + { "mindocid", 1, nullptr, 0 }, + { nullptr, 0, nullptr, 0 } }; enum longopts_enum { LONGOPT_INDEXDIR, @@ -293,7 +294,7 @@ ShowPostingListSubApp::getOptions(int argc, char **argv) _minDocId = atoi(optarg); break; default: - if (optarg != NULL) { + if (optarg != nullptr) { LOG(error, "longopt %s with arg %s", longopts[longopt_index].name, optarg); @@ -683,9 +684,7 @@ DumpWordsSubApp::DumpWordsSubApp() } -DumpWordsSubApp::~DumpWordsSubApp() -{ -} +DumpWordsSubApp::~DumpWordsSubApp() = default; void @@ -708,12 +707,12 @@ DumpWordsSubApp::getOptions(int argc, char **argv) int c; int longopt_index = 0; static struct option longopts[] = { - { "indexdir", 1, NULL, 0 }, - { "field", 1, NULL, 0 }, - { "minnumdocs", 1, NULL, 0 }, - { "verbose", 0, NULL, 0 }, - { "wordnum", 0, NULL, 0 }, - { NULL, 0, NULL, 0 } + { "indexdir", 1, nullptr, 0 }, + { "field", 1, nullptr, 0 }, + { "minnumdocs", 1, nullptr, 0 }, + { "verbose", 0, nullptr, 0 }, + { "wordnum", 0, nullptr, 0 }, + { nullptr, 0, nullptr, 0 } }; enum longopts_enum { LONGOPT_INDEXDIR, @@ -745,7 +744,7 @@ DumpWordsSubApp::getOptions(int argc, char **argv) _showWordNum = true; break; default: - if (optarg != NULL) { + if (optarg != nullptr) { LOG(error, "longopt %s with arg %s", longopts[longopt_index].name, optarg); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java index 171151bfdf4..c7d69d7a36a 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java @@ -10,6 +10,7 @@ import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; import com.yahoo.tensor.TensorType; import com.yahoo.text.Utf8; +import static com.yahoo.searchlib.rankingexpression.Reference.wrapInRankingExpression; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -142,7 +143,7 @@ public class ExpressionFunction { if (shouldGenerateFeature(expr)) { String funcName = "autogenerated_ranking_feature@" + Long.toHexString(symbolCode(key + "=" + binding)); context.addFunctionSerialization(RankingExpression.propertyName(funcName), binding); - binding = "rankingExpression(" + funcName + ")"; + binding = wrapInRankingExpression(funcName); } argumentBindings.put(key, binding); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java index c9f818544e3..c6de04ed755 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java @@ -11,6 +11,7 @@ import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; import com.yahoo.text.Text; +import static com.yahoo.searchlib.rankingexpression.Reference.RANKING_EXPRESSION_WRAPPER; import java.io.File; import java.io.FileNotFoundException; @@ -80,7 +81,7 @@ public class RankingExpression implements Serializable { private String name = ""; private ExpressionNode root; - private final static String RANKEXPRESSION = "rankingExpression("; + private final static String RANKEXPRESSION = RANKING_EXPRESSION_WRAPPER + "("; private final static String RANKINGSCRIPT = ").rankingScript"; private final static String EXPRESSION_NAME = ").expressionName"; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/Reference.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/Reference.java index eaecdf78162..64b251c0cd4 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/Reference.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/Reference.java @@ -120,17 +120,30 @@ public class Reference extends Name implements Comparable<Reference> { return toString(new StringBuilder(), new SerializationContext(), null, null).toString(); } + public static final String RANKING_EXPRESSION_WRAPPER = "rankingExpression"; + + public static String wrapInRankingExpression(String name) { + return RANKING_EXPRESSION_WRAPPER + "(" + name + ")"; + } + + public boolean isSimpleRankingExpressionWrapper() { + return name().equals(RANKING_EXPRESSION_WRAPPER) && isSimple(); + } + public StringBuilder toString(StringBuilder b, SerializationContext context, Deque<String> path, CompositeNode parent) { b.append(name()); if (arguments.expressions().size() > 0) { b.append("("); - for (int i = 0; i < arguments.expressions().size(); i++) { - ExpressionNode e = arguments.expressions().get(i); - e.toString(b, context, path, parent); - if (i+1 < arguments.expressions().size()) { - b.append(','); + if (isSimpleRankingExpressionWrapper()) { + b.append(simpleArgument().get()); + } else { + for (int i = 0; i < arguments.expressions().size(); i++) { + ExpressionNode e = arguments.expressions().get(i); + e.toString(b, context, path, parent); + if (i+1 < arguments.expressions().size()) { + b.append(','); + } } - } b.append(")"); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java index c568db34b4f..ffc05f966fa 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java @@ -32,9 +32,9 @@ import com.yahoo.searchlib.rankingexpression.evaluation.tensoroptimization.Tenso */ public class ExpressionOptimizer { - private GBDTOptimizer gbdtOptimizer = new GBDTOptimizer(); - private GBDTForestOptimizer gbdtForestOptimizer = new GBDTForestOptimizer(); - private TensorOptimizer tensorOptimizer = new TensorOptimizer(); + private final GBDTOptimizer gbdtOptimizer = new GBDTOptimizer(); + private final GBDTForestOptimizer gbdtForestOptimizer = new GBDTForestOptimizer(); + private final TensorOptimizer tensorOptimizer = new TensorOptimizer(); /** Gets an optimizer instance used by this by class name, or null if the optimizer is not known */ public Optimizer getOptimizer(Class<?> clazz) { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java index 25e03c75376..90556bf4b00 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java @@ -57,132 +57,87 @@ public class TensorValue extends Value { @Override public Value or(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.join(((TensorValue)argument).value, (a, b) -> ((a!=0.0) || (b!=0.0)) ? 1.0 : 0.0 )); - else - return new TensorValue(value.map((value) -> ((value!=0.0) || argument.asBoolean()) ? 1 : 0)); + return new TensorValue(value.join(argument.asTensor(), (a, b) -> ((a!=0.0) || (b!=0.0)) ? 1.0 : 0.0 )); } @Override public Value and(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.join(((TensorValue)argument).value, (a, b) -> ((a!=0.0) && (b!=0.0)) ? 1.0 : 0.0 )); - else - return new TensorValue(value.map((value) -> ((value!=0.0) && argument.asBoolean()) ? 1 : 0)); + return new TensorValue(value.join(argument.asTensor(), (a, b) -> ((a!=0.0) && (b!=0.0)) ? 1.0 : 0.0 )); } @Override public Value largerOrEqual(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.largerOrEqual(((TensorValue)argument).value)); - else - return new TensorValue(value.map((value) -> value >= argument.asDouble() ? 1.0 : 0.0)); + return new TensorValue(value.largerOrEqual(argument.asTensor())); } @Override public Value larger(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.larger(((TensorValue)argument).value)); - else - return new TensorValue(value.map((value) -> value > argument.asDouble() ? 1.0 : 0.0)); + return new TensorValue(value.larger(argument.asTensor())); } @Override public Value smallerOrEqual(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.smallerOrEqual(((TensorValue)argument).value)); - else - return new TensorValue(value.map((value) -> value <= argument.asDouble() ? 1.0 : 0.0)); + return new TensorValue(value.smallerOrEqual(argument.asTensor())); } @Override public Value smaller(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.smaller(((TensorValue)argument).value)); - else - return new TensorValue(value.map((value) -> value < argument.asDouble() ? 1.0 : 0.0)); + return new TensorValue(value.smaller(argument.asTensor())); } @Override public Value approxEqual(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.approxEqual(((TensorValue)argument).value)); - else - return new TensorValue(value.map((value) -> DoubleCompatibleValue.approxEqual(value, argument.asDouble()) ? 1.0 : 0.0)); + return new TensorValue(value.approxEqual(argument.asTensor())); } @Override public Value notEqual(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.notEqual(((TensorValue)argument).value)); - else - return new TensorValue(value.map((value) -> value != argument.asDouble() ? 1.0 : 0.0)); + return new TensorValue(value.notEqual(argument.asTensor())); } @Override public Value equal(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.equal(((TensorValue)argument).value)); - else - return new TensorValue(value.map((value) -> value == argument.asDouble() ? 1.0 : 0.0)); + return new TensorValue(value.equal(argument.asTensor())); } @Override public Value add(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.add(((TensorValue)argument).value)); - else - return new TensorValue(value.map((value) -> value + argument.asDouble())); + return new TensorValue(value.add(argument.asTensor())); } @Override public Value subtract(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.subtract(((TensorValue) argument).value)); - else - return new TensorValue(value.map((value) -> value - argument.asDouble())); + return new TensorValue(value.subtract(argument.asTensor())); } @Override public Value multiply(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.multiply(((TensorValue) argument).value)); - else - return new TensorValue(value.map((value) -> value * argument.asDouble())); + return new TensorValue(value.multiply(argument.asTensor())); } @Override public Value divide(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.divide(((TensorValue) argument).value)); - else - return new TensorValue(value.map((value) -> value / argument.asDouble())); + return new TensorValue(value.divide(argument.asTensor())); } @Override public Value modulo(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.fmod(((TensorValue) argument).value)); - else - return new TensorValue(value.map((value) -> value % argument.asDouble())); + return new TensorValue(value.fmod(argument.asTensor())); } @Override public Value power(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.pow(((TensorValue)argument).value)); - else - return new TensorValue(value.map((value) -> Math.pow(value, argument.asDouble()))); + return new TensorValue(value.pow(argument.asTensor())); } public Tensor asTensor() { return value; } @Override public Value function(Function function, Value arg) { - if (arg instanceof TensorValue) + if (function.arity() != 1) return new TensorValue(functionOnTensor(function, arg.asTensor())); else - return new TensorValue(value.map((value) -> function.evaluate(value, arg.asDouble()))); + return new TensorValue(value.map((value) -> function.evaluate(value, 0.0))); } private Tensor functionOnTensor(Function function, Tensor argument) { @@ -195,7 +150,7 @@ public class TensorValue extends Value { case ldexp -> value.ldexp(argument); case bit -> value.bit(argument); case hamming -> value.hamming(argument); - default -> throw new UnsupportedOperationException("Cannot combine two tensors using " + function); + default -> value.join(argument, (a, b) -> function.evaluate(a, b)); }; } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java index 5de2138147e..ed53b82f1d5 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java @@ -114,8 +114,8 @@ public abstract class Value { return new TensorValue(Tensor.from(value)); else if ((value.indexOf('.') == -1) && (value.indexOf('e') == -1) && (value.indexOf('E') == -1)) return new LongValue(Long.parseLong(value)); - else - return new DoubleValue(Double.parseDouble(value)); + else + return new DoubleValue(Double.parseDouble(value)); } public static Value of(Tensor tensor) { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizer.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizer.java index a091cc0287c..33b1824fdeb 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizer.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizer.java @@ -59,7 +59,6 @@ public class TensorOptimizer extends Optimizer { * The ReduceJoin class determines whether or not the arguments are * compatible with the optimization. */ - @SuppressWarnings("unchecked") private ExpressionNode optimizeReduceJoin(ExpressionNode node) { if ( ! (node instanceof TensorFunctionNode)) { return node; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java index 85a12a49958..ec377c6f5d9 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java @@ -8,6 +8,7 @@ import com.yahoo.searchlib.rankingexpression.evaluation.Context; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; +import static com.yahoo.searchlib.rankingexpression.Reference.wrapInRankingExpression; import java.util.ArrayDeque; import java.util.Deque; @@ -95,7 +96,7 @@ public final class ReferenceNode extends CompositeNode { context.addFunctionTypeSerialization(functionName, function.returnType().get()); } path.removeLast(); - return string.append("rankingExpression(").append(functionName).append(')'); + return string.append(wrapInRankingExpression(functionName)); } // Not resolved in this context: output as-is diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java index 7d0c0b98910..c157f44be31 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java @@ -6,6 +6,7 @@ import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; +import static com.yahoo.searchlib.rankingexpression.Reference.wrapInRankingExpression; import java.util.Collection; import java.util.Collections; @@ -97,13 +98,17 @@ public class SerializationContext extends FunctionReferenceContext { /** Adds the serialization of the argument type to a function */ public void addArgumentTypeSerialization(String functionName, String argumentName, TensorType type) { - serializedFunctions.put("rankingExpression(" + functionName + ")." + argumentName + ".type", type.toString()); + serializedFunctions.put(wrapInRankingExpression(functionName) + "." + argumentName + ".type", type.toString()); } /** Adds the serialization of the return type of a function */ public void addFunctionTypeSerialization(String functionName, TensorType type) { if (type.rank() == 0) return; // no explicit type implies scalar (aka rank 0 tensor) - serializedFunctions.put("rankingExpression(" + functionName + ").type", type.toString()); + String key = wrapInRankingExpression(functionName) + ".type"; + var old = serializedFunctions.put(key, type.toString()); + if (old != null && !old.equals(type.toString())) { + throw new IllegalArgumentException("conflicting values for " + key + ": " + old + " != " + type.toString()); + } } @Override diff --git a/searchlib/src/tests/attribute/benchmark/attributebenchmark.cpp b/searchlib/src/tests/attribute/benchmark/attributebenchmark.cpp index 0b90b4eae95..d5ea6fdaffc 100644 --- a/searchlib/src/tests/attribute/benchmark/attributebenchmark.cpp +++ b/searchlib/src/tests/attribute/benchmark/attributebenchmark.cpp @@ -5,7 +5,7 @@ #include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchcommon/attribute/config.h> -#include <vespa/fastos/thread.h> +#include <thread> #include <vespa/vespalib/util/signalhandler.h> #include <iostream> #include "attributesearcher.h" @@ -96,7 +96,6 @@ private: static struct rusage computeDifference(struct rusage & first, struct rusage & second); }; - FastOS_ThreadPool * _threadPool; Config _config; RandomGenerator _rndGen; @@ -129,12 +128,8 @@ private: public: - AttributeBenchmark() : _threadPool(NULL), _config(), _rndGen() {} - ~AttributeBenchmark() { - if (_threadPool != NULL) { - delete _threadPool; - } - } + AttributeBenchmark() : _config(), _rndGen() {} + ~AttributeBenchmark() = default; int main(int argc, char **argv); }; @@ -268,7 +263,7 @@ AttributeBenchmark::benchmarkSearch(const AttributePtr & ptr, const std::vector< } else { searchers.push_back(new AttributeFindSearcher<T>(ptr, values, _config._numQueries)); } - _threadPool->NewThread(searchers.back()); + searchers.back()->start(); } for (uint32_t i = 0; i < searchers.size(); ++i) { @@ -299,7 +294,7 @@ AttributeBenchmark::benchmarkSearchWithUpdater(const AttributePtr & ptr, AttributeUpdaterThread<Vector, T, BT> updater(ptr, values, _rndGen, _config._validate, _config._commitFreq, _config._minValueCount, _config._maxValueCount); - _threadPool->NewThread(&updater); + updater.start(); benchmarkSearch(ptr, values); updater.stop(); updater.join(); @@ -337,8 +332,6 @@ AttributeBenchmark::benchmarkAttribute(const AttributePtr & ptr, const std::vect } else { benchmarkSearchWithUpdater<Vector, T, BT>(ptr, values); } - - _threadPool->Close(); } @@ -569,8 +562,6 @@ AttributeBenchmark::main(int argc, char **argv) dc._attribute = vespalib::string(argv[optind]); - _threadPool = new FastOS_ThreadPool(); - std::cout << "<attribute-benchmark>" << std::endl; init(dc); _config.printXML(); diff --git a/searchlib/src/tests/attribute/benchmark/attributesearcher.h b/searchlib/src/tests/attribute/benchmark/attributesearcher.h index d6e14d6793d..ea2d7190f25 100644 --- a/searchlib/src/tests/attribute/benchmark/attributesearcher.h +++ b/searchlib/src/tests/attribute/benchmark/attributesearcher.h @@ -2,7 +2,6 @@ #pragma once -#include <vespa/searchlib/util/runnable.h> #include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/attribute/search_context.h> @@ -59,7 +58,7 @@ public: }; -class AttributeSearcher : public Runnable +class AttributeSearcher { protected: using AttributePtr = AttributeVector::SP; @@ -67,17 +66,23 @@ protected: const AttributePtr & _attrPtr; vespalib::Timer _timer; AttributeSearcherStatus _status; - + std::thread _thread; + public: - AttributeSearcher(const AttributePtr & attrPtr) : - Runnable(), _attrPtr(attrPtr), _timer(), _status() + AttributeSearcher(const AttributePtr & attrPtr) + : _attrPtr(attrPtr), _timer(), _status(), _thread() { _status._numClients = 1; } - virtual void doRun() override = 0; + virtual ~AttributeSearcher(); + virtual void doRun() = 0; + void start() { _thread = std::thread([this](){doRun();}); } + void join() { _thread.join(); } AttributeSearcherStatus & getStatus() { return _status; } void buildTermQuery(std::vector<char> & buffer, const vespalib::string & index, const char * term, bool prefix = false); }; +AttributeSearcher::~AttributeSearcher() = default; + void AttributeSearcher::buildTermQuery(std::vector<char> & buffer, const vespalib::string & index, const char * term, bool prefix) diff --git a/searchlib/src/tests/attribute/benchmark/attributeupdater.h b/searchlib/src/tests/attribute/benchmark/attributeupdater.h index 88220a8cfb8..ada9c423cd1 100644 --- a/searchlib/src/tests/attribute/benchmark/attributeupdater.h +++ b/searchlib/src/tests/attribute/benchmark/attributeupdater.h @@ -4,7 +4,6 @@ #include <vespa/vespalib/util/hdr_abort.h> #include <vespa/searchlib/util/randomgenerator.h> -#include <vespa/searchlib/util/runnable.h> #include <vespa/searchlib/attribute/attribute.h> #define VALIDATOR_STR(str) #str @@ -153,26 +152,30 @@ template <typename Vector, typename T, typename BT> AttributeUpdater<Vector, T, BT>::~AttributeUpdater() = default; template <typename Vector, typename T, typename BT> -class AttributeUpdaterThread : public AttributeUpdater<Vector, T, BT>, public Runnable +class AttributeUpdaterThread : public AttributeUpdater<Vector, T, BT> { private: using AttributePtr = AttributeVector::SP; - + std::atomic<bool> _done; + std::thread _thread; public: AttributeUpdaterThread(const AttributePtr & attrPtr, const std::vector<T> & values, RandomGenerator & rndGen, bool validate, uint32_t commitFreq, uint32_t minValueCount, uint32_t maxValueCount); ~AttributeUpdaterThread(); - - virtual void doRun() override; + void doRun(); + void start() { _thread = std::thread([this](){doRun();}); } + void stop() { _done = true; } + void join() { _thread.join(); } }; template <typename Vector, typename T, typename BT> AttributeUpdaterThread<Vector, T, BT>::AttributeUpdaterThread(const AttributePtr & attrPtr, const std::vector<T> & values, RandomGenerator & rndGen, bool validate, uint32_t commitFreq, uint32_t minValueCount, uint32_t maxValueCount) - : AttributeUpdater<Vector, T, BT>(attrPtr, values, rndGen, validate, commitFreq, minValueCount, maxValueCount), - Runnable() + : AttributeUpdater<Vector, T, BT>(attrPtr, values, rndGen, validate, commitFreq, minValueCount, maxValueCount), + _done(false), + _thread() {} template <typename Vector, typename T, typename BT> AttributeUpdaterThread<Vector, T, BT>::~AttributeUpdaterThread() = default; diff --git a/searchlib/src/tests/attribute/extendattributes/extendattribute.cpp b/searchlib/src/tests/attribute/extendattributes/extendattribute.cpp index 35dd0351f34..a44965ffb31 100644 --- a/searchlib/src/tests/attribute/extendattributes/extendattribute.cpp +++ b/searchlib/src/tests/attribute/extendattributes/extendattribute.cpp @@ -101,9 +101,11 @@ void ExtendAttributeTest::testExtendString(Attribute & attr) EXPECT_EQ(docId, 0u); EXPECT_EQ(attr.getNumDocs(), 1u); attr.add("1.7", 10); - EXPECT_EQ(std::string(attr.getString(0, NULL, 0)), "1.7"); + auto buf = attr.get_raw(0); + EXPECT_EQ(std::string(buf.data(), buf.size()), "1.7"); attr.add("2.3", 20); - EXPECT_EQ(std::string(attr.getString(0, NULL, 0)), attr.hasMultiValue() ? "1.7" : "2.3"); + buf = attr.get_raw(0); + EXPECT_EQ(std::string(buf.data(), buf.size()), attr.hasMultiValue() ? "1.7" : "2.3"); if (attr.hasMultiValue()) { AttributeVector::WeightedString v[2]; EXPECT_EQ((static_cast<AttributeVector &>(attr)).get(0, v, 2), 2u); @@ -118,7 +120,8 @@ void ExtendAttributeTest::testExtendString(Attribute & attr) EXPECT_EQ(docId, 1u); EXPECT_EQ(attr.getNumDocs(), 2u); attr.add("3.6", 30); - EXPECT_EQ(std::string(attr.getString(1, NULL, 0)), "3.6"); + buf = attr.get_raw(1); + EXPECT_EQ(std::string(buf.data(), buf.size()), "3.6"); if (attr.hasMultiValue()) { AttributeVector::WeightedString v[1]; EXPECT_EQ((static_cast<AttributeVector &>(attr)).get(1, v, 1), 1u); diff --git a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp index 0d2ce048111..9e65dfcfc07 100644 --- a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp +++ b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp @@ -258,9 +258,10 @@ SingleStringAttrFixture::~SingleStringAttrFixture() = default; TEST_F("Single-valued string attribute values can be retrieved via reference", SingleStringAttrFixture) { - char buf[64]; - EXPECT_EQUAL(vespalib::string("foo"), f.get_imported_attr()->getString(DocId(2), buf, sizeof(buf))); - EXPECT_EQUAL(vespalib::string("bar"), f.get_imported_attr()->getString(DocId(4), buf, sizeof(buf))); + auto buf = f.get_imported_attr()->get_raw(DocId(2)); + EXPECT_EQUAL(vespalib::stringref("foo"), vespalib::stringref(buf.data(), buf.size())); + buf = f.get_imported_attr()->get_raw(DocId(4)); + EXPECT_EQUAL(vespalib::stringref("bar"), vespalib::stringref(buf.data(), buf.size())); } TEST_F("getEnum() returns target vector enum via reference", SingleStringAttrFixture) diff --git a/searchlib/src/tests/attribute/postinglist/postinglist.cpp b/searchlib/src/tests/attribute/postinglist/postinglist.cpp index 3f07faa159f..7d2a89b6e5b 100644 --- a/searchlib/src/tests/attribute/postinglist/postinglist.cpp +++ b/searchlib/src/tests/attribute/postinglist/postinglist.cpp @@ -11,6 +11,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <set> #include <map> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP("postinglist_test"); diff --git a/searchlib/src/tests/attribute/raw_attribute/CMakeLists.txt b/searchlib/src/tests/attribute/raw_attribute/CMakeLists.txt new file mode 100644 index 00000000000..21e34f42193 --- /dev/null +++ b/searchlib/src/tests/attribute/raw_attribute/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_raw_attribute_test_app TEST + SOURCES + raw_attribute_test.cpp + DEPENDS + searchlib + GTest::GTest +) +vespa_add_test(NAME searchlib_raw_attribute_test_app COMMAND searchlib_raw_attribute_test_app) diff --git a/searchlib/src/tests/attribute/raw_attribute/raw_attribute_test.cpp b/searchlib/src/tests/attribute/raw_attribute/raw_attribute_test.cpp new file mode 100644 index 00000000000..82e4fd065cf --- /dev/null +++ b/searchlib/src/tests/attribute/raw_attribute/raw_attribute_test.cpp @@ -0,0 +1,92 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchlib/attribute/single_raw_attribute.h> +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchcommon/attribute/config.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <memory> + +using search::AttributeFactory; +using search::AttributeVector; +using search::attribute::BasicType; +using search::attribute::CollectionType; +using search::attribute::Config; +using search::attribute::SingleRawAttribute; +using vespalib::ConstArrayRef; + + +std::vector<char> empty; +vespalib::string hello("hello"); +vespalib::ConstArrayRef<char> raw_hello(hello.c_str(), hello.size()); + +std::vector<char> as_vector(vespalib::stringref value) { + return {value.data(), value.data() + value.size()}; +} + +std::vector<char> as_vector(vespalib::ConstArrayRef<char> value) { + return {value.data(), value.data() + value.size()}; +} + +class RawAttributeTest : public ::testing::Test +{ +protected: + std::shared_ptr<AttributeVector> _attr; + SingleRawAttribute* _raw; + + RawAttributeTest(); + ~RawAttributeTest() override; + std::vector<char> get_raw(uint32_t docid); +}; + + +RawAttributeTest::RawAttributeTest() + : ::testing::Test(), + _attr(), + _raw(nullptr) +{ + Config cfg(BasicType::RAW, CollectionType::SINGLE); + _attr = AttributeFactory::createAttribute("raw", cfg); + _raw = &dynamic_cast<SingleRawAttribute&>(*_attr); + _attr->addReservedDoc(); +} + +RawAttributeTest::~RawAttributeTest() = default; + +std::vector<char> +RawAttributeTest::get_raw(uint32_t docid) +{ + return as_vector(_raw->get_raw(docid)); +} + +TEST_F(RawAttributeTest, can_set_and_clear_value) +{ + EXPECT_TRUE(_attr->addDocs(10)); + _attr->commit(); + EXPECT_EQ(empty, get_raw(1)); + _raw->set_raw(1, raw_hello); + EXPECT_EQ(as_vector(hello), get_raw(1)); + _attr->clearDoc(1); + EXPECT_EQ(empty, get_raw(1)); +} + +TEST_F(RawAttributeTest, implements_serialize_for_sort) { + vespalib::string long_hello("hello, is there anybody out there"); + vespalib::ConstArrayRef<char> raw_long_hello(long_hello.c_str(), long_hello.size()); + uint8_t buf[8]; + memset(buf, 0, sizeof(buf)); + _attr->addDocs(10); + _attr->commit(); + EXPECT_EQ(0, _attr->serializeForAscendingSort(1, buf, sizeof(buf))); + EXPECT_EQ(0, _attr->serializeForDescendingSort(1, buf, sizeof(buf))); + _raw->set_raw(1, raw_hello); + EXPECT_EQ(5, _attr->serializeForAscendingSort(1, buf, sizeof(buf))); + EXPECT_EQ(0, memcmp("hello", buf, 5)); + EXPECT_EQ(5, _attr->serializeForDescendingSort(1, buf, sizeof(buf))); + uint8_t expected [] = {0xff-'h', 0xff-'e', 0xff-'l', 0xff-'l', 0xff-'o'}; + EXPECT_EQ(0, memcmp(expected, buf, 5)); + _raw->set_raw(1, raw_long_hello); + EXPECT_EQ(-1, _attr->serializeForAscendingSort(1, buf, sizeof(buf))); + EXPECT_EQ(-1, _attr->serializeForDescendingSort(1, buf, sizeof(buf))); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/attribute/raw_buffer_type_mapper/CMakeLists.txt b/searchlib/src/tests/attribute/raw_buffer_type_mapper/CMakeLists.txt new file mode 100644 index 00000000000..c860770536d --- /dev/null +++ b/searchlib/src/tests/attribute/raw_buffer_type_mapper/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_raw_buffer_type_mapper_test_app TEST + SOURCES + raw_buffer_type_mapper_test.cpp + DEPENDS + searchlib + GTest::GTest +) +vespa_add_test(NAME searchlib_raw_buffer_type_mapper_test_app COMMAND searchlib_raw_buffer_type_mapper_test_app) diff --git a/searchlib/src/tests/attribute/raw_buffer_type_mapper/raw_buffer_type_mapper_test.cpp b/searchlib/src/tests/attribute/raw_buffer_type_mapper/raw_buffer_type_mapper_test.cpp new file mode 100644 index 00000000000..74ec839670e --- /dev/null +++ b/searchlib/src/tests/attribute/raw_buffer_type_mapper/raw_buffer_type_mapper_test.cpp @@ -0,0 +1,115 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchlib/attribute/raw_buffer_type_mapper.h> +#include <vespa/vespalib/gtest/gtest.h> + +using search::attribute::RawBufferTypeMapper; + +constexpr double default_grow_factor = 1.03; + +class RawBufferTypeMapperTest : public testing::Test +{ +protected: + RawBufferTypeMapper _mapper; + RawBufferTypeMapperTest(); + ~RawBufferTypeMapperTest() override; + std::vector<size_t> get_array_sizes(uint32_t num_array_sizes); + std::vector<size_t> get_large_array_sizes(uint32_t num_large_arrays); + void select_type_ids(std::vector<size_t> array_sizes); + void setup_mapper(uint32_t max_small_buffer_type_id, double grow_factor); + static uint32_t calc_max_small_array_type_id(double grow_factor); +}; + +RawBufferTypeMapperTest::RawBufferTypeMapperTest() + : testing::Test(), + _mapper(5, default_grow_factor) +{ +} + +RawBufferTypeMapperTest::~RawBufferTypeMapperTest() = default; + +void +RawBufferTypeMapperTest::setup_mapper(uint32_t max_small_buffer_type_id, double grow_factor) +{ + _mapper = RawBufferTypeMapper(max_small_buffer_type_id, grow_factor); +} + +std::vector<size_t> +RawBufferTypeMapperTest::get_array_sizes(uint32_t num_array_sizes) +{ + std::vector<size_t> array_sizes; + for (uint32_t type_id = 1; type_id <= num_array_sizes; ++type_id) { + array_sizes.emplace_back(_mapper.get_array_size(type_id)); + } + return array_sizes; +} + +std::vector<size_t> +RawBufferTypeMapperTest::get_large_array_sizes(uint32_t num_large_array_sizes) +{ + setup_mapper(num_large_array_sizes * 100, default_grow_factor); + std::vector<size_t> result; + for (uint32_t i = 0; i < num_large_array_sizes; ++i) { + uint32_t type_id = (i + 1) * 100; + auto array_size = _mapper.get_array_size(type_id); + result.emplace_back(array_size); + EXPECT_EQ(type_id, _mapper.get_type_id(array_size)); + EXPECT_EQ(type_id, _mapper.get_type_id(array_size - 1)); + if (i + 1 == num_large_array_sizes) { + EXPECT_EQ(0u, _mapper.get_type_id(array_size + 1)); + } else { + EXPECT_EQ(type_id + 1, _mapper.get_type_id(array_size + 1)); + } + } + return result; +} + +void +RawBufferTypeMapperTest::select_type_ids(std::vector<size_t> array_sizes) +{ + uint32_t type_id = 0; + for (auto array_size : array_sizes) { + ++type_id; + EXPECT_EQ(type_id, _mapper.get_type_id(array_size)); + EXPECT_EQ(type_id, _mapper.get_type_id(array_size - 1)); + if (array_size == array_sizes.back()) { + // Fallback to indirect storage, using type id 0 + EXPECT_EQ(0u, _mapper.get_type_id(array_size + 1)); + } else { + EXPECT_EQ(type_id + 1, _mapper.get_type_id(array_size + 1)); + } + } +} + +uint32_t +RawBufferTypeMapperTest::calc_max_small_array_type_id(double grow_factor) +{ + RawBufferTypeMapper mapper(1000, grow_factor); + return mapper.get_max_small_array_type_id(1000); +} + +TEST_F(RawBufferTypeMapperTest, array_sizes_are_calculated) +{ + EXPECT_EQ((std::vector<size_t>{8, 12, 16, 20, 24}), get_array_sizes(5)); +} + +TEST_F(RawBufferTypeMapperTest, type_ids_are_selected) +{ + select_type_ids({8, 12, 16, 20, 24}); +} + +TEST_F(RawBufferTypeMapperTest, large_arrays_grows_exponentially) +{ + EXPECT_EQ((std::vector<size_t>{1148, 22796, 438572, 8429384}), get_large_array_sizes(4)); +} + +TEST_F(RawBufferTypeMapperTest, avoid_array_size_overflow) +{ + EXPECT_EQ(29, calc_max_small_array_type_id(2.0)); + EXPECT_EQ(379, calc_max_small_array_type_id(1.05)); + EXPECT_EQ(468, calc_max_small_array_type_id(1.04)); + EXPECT_EQ(610, calc_max_small_array_type_id(1.03)); + EXPECT_EQ(892, calc_max_small_array_type_id(1.02)); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp b/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp index b5101f1ea58..e356187a19f 100644 --- a/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp +++ b/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp @@ -15,6 +15,7 @@ #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/test/insertion_operators.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP("reference_attribute_test"); diff --git a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp index 20373fbb3a9..3d4edcd4aea 100644 --- a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp +++ b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp @@ -303,7 +303,7 @@ testDefaultValueOnAddDoc(AttributeVector & v) EXPECT_EQUAL(1u, doc); EXPECT_EQUAL(2u, v.getNumDocs()); EXPECT_TRUE( IEnumStore::Index(EntryRef(v.getEnum(doc))).valid() ); - EXPECT_EQUAL(0u, strlen(v.getString(doc, NULL, 0))); + EXPECT_EQUAL(0u, v.get_raw(doc).size()); } template <typename Attribute> @@ -339,6 +339,9 @@ testSingleValue(Attribute & svsa, Config &cfg) for (uint32_t j = i - 9; j <= i; ++j) { snprintf(tmp, sizeof(tmp), "enum%u", j % 10); EXPECT_TRUE( strcmp(t = v.get(j), tmp) == 0 ); + auto raw = v.get_raw(j); + EXPECT_EQUAL(strlen(tmp), raw.size()); + EXPECT_EQUAL(0, memcmp(raw.data(), tmp, raw.size())); e1 = v.getEnum(j); EXPECT_TRUE( v.findEnum(t, e2) ); EXPECT_TRUE( e1 == e2 ); diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 2f51459ebfa..28c50891225 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -26,9 +26,11 @@ #include <vespa/searchlib/util/bufferwriter.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/document/base/exceptions.h> +#include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/value_codec.h> #include <vespa/eval/eval/test/value_compare.h> #include <vespa/fastos/file.h> #include <filesystem> @@ -60,7 +62,9 @@ using search::tensor::PrepareResult; using search::tensor::SerializedFastValueAttribute; using search::tensor::TensorAttribute; using search::tensor::VectorBundle; +using vespalib::SharedStringRepo; using vespalib::datastore::CompactionStrategy; +using vespalib::eval::FastValueBuilderFactory; using vespalib::eval::CellType; using vespalib::eval::SimpleValue; using vespalib::eval::TensorSpec; @@ -76,7 +80,17 @@ vespalib::string vec_2d_spec("tensor(x[2])"); vespalib::string vec_mixed_2d_spec("tensor(a{},x[2])"); Value::UP createTensor(const TensorSpec &spec) { - return SimpleValue::from_spec(spec); + return value_from_spec(spec, FastValueBuilderFactory::get()); +} + +std::vector<vespalib::string> +to_string_labels(vespalib::ConstArrayRef<vespalib::string_id> labels) +{ + std::vector<vespalib::string> result; + for (auto& label : labels) { + result.emplace_back(SharedStringRepo::Handle::string_from_id(label)); + } + return result; } TensorSpec @@ -569,6 +583,7 @@ struct Fixture { void testCompaction(); void testTensorTypeFileHeaderTag(); void testEmptyTensor(); + void testSerializedTensorRef(); void testOnHoldAccounting(); void test_populate_address_space_usage(); void test_mmap_file_allocator(); @@ -776,6 +791,44 @@ Fixture::testEmptyTensor() } void +Fixture::testSerializedTensorRef() +{ + const TensorAttribute &tensorAttr = *_tensorAttr; + if (_traits.use_dense_tensor_attribute || _traits.use_direct_tensor_attribute) { + EXPECT_FALSE(tensorAttr.supports_get_serialized_tensor_ref()); + return; + } + EXPECT_TRUE(tensorAttr.supports_get_serialized_tensor_ref()); + if (_denseTensors) { + set_tensor(3, expDenseTensor3()); + } else { + set_tensor(3, TensorSpec(sparseSpec) + .add({{"x", "one"}, {"y", "two"}}, 11) + .add({{"x", "three"}, {"y", "four"}}, 17)); + } + auto ref = tensorAttr.get_serialized_tensor_ref(3); + auto vectors = ref.get_vectors(); + if (_denseTensors) { + EXPECT_EQUAL(1u, vectors.subspaces()); + auto cells = vectors.cells(0).typify<double>(); + auto labels = ref.get_labels(0); + EXPECT_EQUAL(0u, labels.size()); + EXPECT_EQUAL((std::vector<double>{0.0, 11.0, 0.0, 0.0, 0.0, 0.0}), (std::vector<double>{ cells.begin(), cells.end() })); + } else { + EXPECT_EQUAL(2u, vectors.subspaces()); + auto cells = vectors.cells(0).typify<double>(); + auto labels = ref.get_labels(0); + EXPECT_EQUAL((std::vector<vespalib::string>{"one", "two"}), to_string_labels(labels)); + EXPECT_EQUAL((std::vector<double>{11.0}), (std::vector<double>{ cells.begin(), cells.end() })); + cells = vectors.cells(1).typify<double>(); + labels = ref.get_labels(1); + EXPECT_EQUAL((std::vector<vespalib::string>{"three", "four"}), to_string_labels(labels)); + EXPECT_EQUAL((std::vector<double>{17.0}), (std::vector<double>{ cells.begin(), cells.end() })); + } + TEST_DO(clearTensor(3)); +} + +void Fixture::testOnHoldAccounting() { { @@ -829,6 +882,7 @@ void testAll(MakeFixture &&f) TEST_DO(f()->testCompaction()); TEST_DO(f()->testTensorTypeFileHeaderTag()); TEST_DO(f()->testEmptyTensor()); + TEST_DO(f()->testSerializedTensorRef()); TEST_DO(f()->testOnHoldAccounting()); TEST_DO(f()->test_populate_address_space_usage()); TEST_DO(f()->test_mmap_file_allocator()); diff --git a/searchlib/src/tests/bitcompression/expgolomb/expgolomb_test.cpp b/searchlib/src/tests/bitcompression/expgolomb/expgolomb_test.cpp index 5bb36a8462e..9a726f9d8a6 100644 --- a/searchlib/src/tests/bitcompression/expgolomb/expgolomb_test.cpp +++ b/searchlib/src/tests/bitcompression/expgolomb/expgolomb_test.cpp @@ -5,6 +5,7 @@ #include <vespa/vespalib/util/size_literals.h> #include <vector> #include <algorithm> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP("expglomb_test"); diff --git a/searchlib/src/tests/common/location_iterator/location_iterator_test.cpp b/searchlib/src/tests/common/location_iterator/location_iterator_test.cpp index 44a2f1697cd..bf372c0e62f 100644 --- a/searchlib/src/tests/common/location_iterator/location_iterator_test.cpp +++ b/searchlib/src/tests/common/location_iterator/location_iterator_test.cpp @@ -9,6 +9,7 @@ #include <vespa/searchlib/common/locationiterators.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/gtest/gtest.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP("location_iterator_test"); diff --git a/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp b/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp index f1729f21f39..408cf370c59 100644 --- a/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp +++ b/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp @@ -16,6 +16,7 @@ #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/vespalib/util/signalhandler.h> #include <sstream> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP("pagedict4test"); diff --git a/searchlib/src/tests/engine/proto_rpc_adapter/proto_rpc_adapter_test.cpp b/searchlib/src/tests/engine/proto_rpc_adapter/proto_rpc_adapter_test.cpp index 9c5e000c314..09fb0981a24 100644 --- a/searchlib/src/tests/engine/proto_rpc_adapter/proto_rpc_adapter_test.cpp +++ b/searchlib/src/tests/engine/proto_rpc_adapter/proto_rpc_adapter_test.cpp @@ -124,8 +124,8 @@ TEST_F(ProtoRpcAdapterTest, require_that_plain_rpc_ping_works) { req->SetMethodName("frt.rpc.ping"); target->InvokeSync(req, 60.0); EXPECT_TRUE(req->CheckReturnTypes("")); - req->SubRef(); - target->SubRef(); + req->internal_subref(); + target->internal_subref(); } TEST_F(ProtoRpcAdapterTest, require_that_proto_rpc_search_works) { @@ -145,9 +145,9 @@ TEST_F(ProtoRpcAdapterTest, require_that_proto_rpc_search_works) { EXPECT_EQ(std::string(rpc->GetErrorMessage()), std::string("Server not online")); adapter.set_online(); } - rpc->SubRef(); + rpc->internal_subref(); } - target->SubRef(); + target->internal_subref(); SearchProtocolMetrics &metrics = adapter.metrics(); EXPECT_EQ(metrics.query().latency.getCount(), 2); EXPECT_GT(metrics.query().latency.getTotal(), 0.0); @@ -180,9 +180,9 @@ TEST_F(ProtoRpcAdapterTest, require_that_proto_rpc_getDocsums_works) { EXPECT_EQ(std::string(rpc->GetErrorMessage()), std::string("Server not online")); adapter.set_online(); } - rpc->SubRef(); + rpc->internal_subref(); } - target->SubRef(); + target->internal_subref(); SearchProtocolMetrics &metrics = adapter.metrics(); EXPECT_EQ(metrics.query().latency.getCount(), 0); EXPECT_EQ(metrics.docsum().latency.getCount(), 2); @@ -208,9 +208,9 @@ TEST_F(ProtoRpcAdapterTest, require_that_proto_rpc_ping_works) { EXPECT_EQ(std::string(rpc->GetErrorMessage()), std::string("Server not online")); adapter.set_online(); } - rpc->SubRef(); + rpc->internal_subref(); } - target->SubRef(); + target->internal_subref(); SearchProtocolMetrics &metrics = adapter.metrics(); EXPECT_EQ(metrics.query().latency.getCount(), 0); EXPECT_EQ(metrics.docsum().latency.getCount(), 0); diff --git a/searchlib/src/tests/features/closest/CMakeLists.txt b/searchlib/src/tests/features/closest/CMakeLists.txt new file mode 100644 index 00000000000..71572c5e5a2 --- /dev/null +++ b/searchlib/src/tests/features/closest/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +vespa_add_executable(searchlib_closest_test_app TEST + SOURCES + closest_test.cpp + DEPENDS + searchlib + searchlib_test +) +vespa_add_test(NAME searchlib_closest_test_app COMMAND searchlib_closest_test_app) diff --git a/searchlib/src/tests/features/closest/closest_test.cpp b/searchlib/src/tests/features/closest/closest_test.cpp new file mode 100644 index 00000000000..c53e627b528 --- /dev/null +++ b/searchlib/src/tests/features/closest/closest_test.cpp @@ -0,0 +1,208 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/searchlib/features/closest_feature.h> +#include <vespa/searchlib/features/setup.h> +#include <vespa/searchlib/fef/test/dummy_dependency_handler.h> +#include <vespa/searchlib/fef/test/labels.h> +#include <vespa/searchlib/test/features/distance_closeness_fixture.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/stllike/asciistream.h> + +using search::feature_t; +using search::features::test::BlueprintFactoryFixture; +using search::features::test::DistanceClosenessFixture; +using search::features::test::FeatureDumpFixture; +using search::features::test::IndexEnvironmentFixture; +using search::features::ClosestBlueprint; +using vespalib::eval::FastValueBuilderFactory; +using vespalib::eval::TensorSpec; +using vespalib::eval::Value; +using vespalib::eval::spec_from_value; +using vespalib::eval::value_from_spec; + +const vespalib::string field_and_label_feature_name("closest(bar,nns)"); +const vespalib::string field_feature_name("closest(bar)"); + +const vespalib::string dense_tensor_type("tensor(x[2])"); +const vespalib::string mixed_tensor_type("tensor(a{},x[2])"); +const vespalib::string sparse_tensor_type("tensor(a{})"); + +TensorSpec no_subspace(sparse_tensor_type); +TensorSpec subspace_a = TensorSpec::from_expr("tensor(a{}):{{a:\"a\"}:1}"); +TensorSpec subspace_b = TensorSpec::from_expr("tensor(a{}):{{a:\"b\"}:1}"); + +TensorSpec doc_tensor = TensorSpec::from_expr("tensor(a{},x[2]):{{a:\"a\",x:0}:3,{a:\"a\",x:1}:10,{a:\"b\",x:0}:5,{a:\"b\",x:1}:10}"); + +using RankFixture = DistanceClosenessFixture; + +TensorSpec get_spec(RankFixture& f, uint32_t docid) { + return spec_from_value(f.getObject(docid).get()); +} + +struct TestParam +{ + vespalib::string _name; + bool _direct_tensor; + TestParam(vespalib::string name, bool direct_tensor) + : _name(std::move(name)), + _direct_tensor(direct_tensor) + { + } + ~TestParam(); +}; + +TestParam::~TestParam() = default; + +std::ostream& operator<<(std::ostream& os, const TestParam param) +{ + os << param._name; + return os; +} + +void +assert_setup(vespalib::string field_name, + bool exp_setup_result, + std::optional<vespalib::string> attr_type_spec, + std::optional<vespalib::string> label) +{ + vespalib::asciistream feature_name; + std::vector<vespalib::string> setup_args; + ClosestBlueprint f1; + IndexEnvironmentFixture f2; + DummyDependencyHandler deps(f1); + setup_args.emplace_back(field_name); + feature_name << f1.getBaseName() << "(" << field_name; + if (label.has_value()) { + feature_name << "," << label.value(); + setup_args.emplace_back(label.value()); + } + feature_name << ")"; + f1.setName(feature_name.str()); + if (attr_type_spec.has_value()) { + search::fef::indexproperties::type::Attribute::set(f2.indexEnv.getProperties(), field_name, attr_type_spec.value()); + } + EXPECT_EQ(exp_setup_result, static_cast<Blueprint&>(f1).setup(f2.indexEnv, setup_args)); +} + +class ClosestTest : public ::testing::TestWithParam<TestParam> +{ +protected: + ClosestTest(); + ~ClosestTest(); + bool direct_tensor() const noexcept { return GetParam()._direct_tensor; } + void assert_closest(const Labels& labels, const vespalib::string& feature_name, const vespalib::string& query_tensor, const TensorSpec& exp_spec); + void assert_closest(const Labels& labels, const vespalib::string& feature_name, const std::vector<TensorSpec>& exp_specs); +}; + +ClosestTest::ClosestTest() + : testing::TestWithParam<TestParam>() +{ +} + +ClosestTest::~ClosestTest() = default; + +void +ClosestTest::assert_closest(const Labels& labels, const vespalib::string& feature_name, const vespalib::string& query_tensor, const TensorSpec& exp_spec) +{ + RankFixture f(mixed_tensor_type, direct_tensor(), 0, 1, labels, feature_name, + dense_tensor_type + ":" + query_tensor); + ASSERT_FALSE(f.failed()); + SCOPED_TRACE(query_tensor); + f.set_attribute_tensor(9, doc_tensor); + EXPECT_EQ(exp_spec, get_spec(f, 9)); +} + +void +ClosestTest::assert_closest(const Labels& labels, const vespalib::string& feature_name, const std::vector<TensorSpec>& exp_specs) +{ + assert_closest(labels, feature_name, "[9,10]", exp_specs[0]); + assert_closest(labels, feature_name, "[1,10]", exp_specs[1]); +} + +INSTANTIATE_TEST_SUITE_P(ClosestMultiTest, + ClosestTest, + testing::Values(TestParam("Serialized", false), + TestParam("Direct", true)), + testing::PrintToStringParamName()); + +TEST(ClosestTest, require_that_blueprint_can_be_created_from_factory) +{ + BlueprintFactoryFixture f; + auto bp = f.factory.createBlueprint("closest"); + EXPECT_TRUE(bp); + EXPECT_TRUE(dynamic_cast<ClosestBlueprint*>(bp.get()) != 0); +} + +TEST(ClosestTest, require_that_no_features_are_dumped) +{ + ClosestBlueprint f1; + IndexEnvironmentFixture f2; + FeatureDumpFixture f3; + f1.visitDumpFeatures(f2.indexEnv, f3); +} + +TEST(ClosestTest, require_that_setup_fails_for_unknown_field) +{ + assert_setup("random_field", false, mixed_tensor_type, std::nullopt); +} + +TEST(ClosestTest, require_that_setup_fails_if_field_type_is_not_attribute) +{ + assert_setup("ibar", false, sparse_tensor_type, std::nullopt); +} + +TEST(ClosestTest, require_that_setup_fails_if_field_data_type_is_not_tensor) +{ + assert_setup("foo", false, sparse_tensor_type, std::nullopt); +} + +TEST(ClosestTest, require_that_setup_can_be_done_on_random_label) +{ + assert_setup("bar", true, mixed_tensor_type, "random_label"); +} + +TEST(ClosestTest, require_that_setup_fails_if_tensor_type_is_missing) +{ + assert_setup("bar", false, std::nullopt, std::nullopt); +} + +TEST(ClosestTest, require_that_setup_fails_if_tensor_type_is_dense) +{ + assert_setup("bar", false, dense_tensor_type, std::nullopt); +} + +TEST(ClosestTest, require_that_setup_fails_if_tensor_type_is_sparse) +{ + assert_setup("bar", false, sparse_tensor_type, std::nullopt); +} + +TEST_P(ClosestTest, require_that_no_label_gives_empty_result) +{ + NoLabel f; + assert_closest(f, field_and_label_feature_name, {no_subspace, no_subspace}); +} + +TEST_P(ClosestTest, require_that_unrelated_label_gives_empty_result) +{ + SingleLabel f("unrelated", 1); + assert_closest(f, field_and_label_feature_name, {no_subspace, no_subspace}); +} + +TEST_P(ClosestTest, closest_using_field_setup) +{ + NoLabel f; + assert_closest(f, field_feature_name, {subspace_b, subspace_a}); +} + +TEST_P(ClosestTest, closest_using_field_and_label_setup) +{ + SingleLabel f("nns", 1); + assert_closest(f, field_and_label_feature_name, {subspace_b, subspace_a}); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/features/nns_closeness/nns_closeness_test.cpp b/searchlib/src/tests/features/nns_closeness/nns_closeness_test.cpp index e67370d48f6..8cb060c08e4 100644 --- a/searchlib/src/tests/features/nns_closeness/nns_closeness_test.cpp +++ b/searchlib/src/tests/features/nns_closeness/nns_closeness_test.cpp @@ -7,7 +7,7 @@ #include <vespa/searchlib/fef/test/labels.h> #include <vespa/searchlib/test/features/distance_closeness_fixture.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/stringfmt.h> using search::feature_t; @@ -23,60 +23,97 @@ const vespalib::string fieldFeatureName("closeness(bar)"); using RankFixture = DistanceClosenessFixture; -TEST_F("require that blueprint can be created from factory", BlueprintFactoryFixture) { +TEST(NnsClosenessTest, require_that_blueprint_can_be_created_from_factory) +{ + BlueprintFactoryFixture f; Blueprint::SP bp = f.factory.createBlueprint("closeness"); EXPECT_TRUE(bp.get() != 0); EXPECT_TRUE(dynamic_cast<ClosenessBlueprint*>(bp.get()) != 0); } -TEST_FFF("require that no features are dumped", ClosenessBlueprint, IndexEnvironmentFixture, FeatureDumpFixture) { +TEST(NnsClosenessTest, require_that_no_features_are_dumped) +{ + ClosenessBlueprint f1; + IndexEnvironmentFixture f2; + FeatureDumpFixture f3; f1.visitDumpFeatures(f2.indexEnv, f3); } -TEST_FF("require that setup can be done on random label", ClosenessBlueprint, IndexEnvironmentFixture) { +TEST(NnsClosenessTest, require_that_setup_can_be_done_on_random_label) +{ + ClosenessBlueprint f1; + IndexEnvironmentFixture f2; DummyDependencyHandler deps(f1); f1.setName(vespalib::make_string("%s(label,random_label)", f1.getBaseName().c_str())); EXPECT_TRUE(static_cast<Blueprint&>(f1).setup(f2.indexEnv, std::vector<vespalib::string>{"label", "random_label"})); } -TEST_FF("require that no label gives 0 closeness", NoLabel(), RankFixture(2, 2, f1, labelFeatureName)) { - EXPECT_EQUAL(0.0, f2.getScore(10)); +TEST(NnsClosenessTest, require_that_no_label_gives_0_closeness) +{ + NoLabel f1; + RankFixture f2(2, 2, f1, labelFeatureName); + ASSERT_FALSE(f2.failed()); + EXPECT_EQ(0.0, f2.getScore(10)); } -TEST_FF("require that unrelated label gives 0 closeness", SingleLabel("unrelated", 1), RankFixture(2, 2, f1, labelFeatureName)) { - EXPECT_EQUAL(0.0, f2.getScore(10)); +TEST(NnsClosenessTest, require_that_unrelated_label_gives_0_closeness) +{ + SingleLabel f1("unrelated", 1); + RankFixture f2(2, 2, f1, labelFeatureName); + ASSERT_FALSE(f2.failed()); + EXPECT_EQ(0.0, f2.getScore(10)); } -TEST_FF("require that labeled item raw score can be obtained", SingleLabel("nns", 1), RankFixture(2, 2, f1, labelFeatureName)) { +TEST(NnsClosenessTest, require_that_labeled_item_raw_score_can_be_obtained) +{ + SingleLabel f1("nns", 1); + RankFixture f2(2, 2, f1, labelFeatureName); + ASSERT_FALSE(f2.failed()); f2.setFooScore(0, 10, 5.0); - EXPECT_EQUAL(1/(1+5.0), f2.getScore(10)); + EXPECT_EQ(1/(1+5.0), f2.getScore(10)); } -TEST_FF("require that field raw score can be obtained", NoLabel(), RankFixture(2, 2, f1, fieldFeatureName)) { +TEST(NnsClosenessTest, require_that_field_raw_score_can_be_obtained) +{ + NoLabel f1; + RankFixture f2(2, 2, f1, fieldFeatureName); + ASSERT_FALSE(f2.failed()); f2.setBarScore(0, 10, 5.0); - EXPECT_EQUAL(1/(1+5.0), f2.getScore(10)); + EXPECT_EQ(1/(1+5.0), f2.getScore(10)); } -TEST_FF("require that other raw scores are ignored", SingleLabel("nns", 2), RankFixture(2, 2, f1, labelFeatureName)) { +TEST(NnsClosenessTest, require_that_other_raw_scores_are_ignored) +{ + SingleLabel f1("nns", 2); + RankFixture f2(2, 2, f1, labelFeatureName); + ASSERT_FALSE(f2.failed()); f2.setFooScore(0, 10, 1.0); f2.setFooScore(1, 10, 2.0); f2.setBarScore(0, 10, 5.0); f2.setBarScore(1, 10, 6.0); - EXPECT_EQUAL(1/(1+2.0), f2.getScore(10)); + EXPECT_EQ(1/(1+2.0), f2.getScore(10)); } -TEST_FF("require that the correct raw score is used", NoLabel(), RankFixture(2, 2, f1, fieldFeatureName)) { +TEST(NnsClosenessTest, require_that_the_correct_raw_score_is_used) +{ + NoLabel f1; + RankFixture f2(2, 2, f1, fieldFeatureName); + ASSERT_FALSE(f2.failed()); f2.setFooScore(0, 10, 3.0); f2.setFooScore(1, 10, 4.0); f2.setBarScore(0, 10, 8.0); f2.setBarScore(1, 10, 7.0); - EXPECT_EQUAL(1/(1+7.0), f2.getScore(10)); + EXPECT_EQ(1/(1+7.0), f2.getScore(10)); } -TEST_FF("require that stale data is ignored", SingleLabel("nns", 2), RankFixture(2, 2, f1, labelFeatureName)) { +TEST(NnsClosenessTest, require_that_stale_data_is_ignored) +{ + SingleLabel f1("nns", 2); + RankFixture f2(2, 2, f1, labelFeatureName); + ASSERT_FALSE(f2.failed()); f2.setFooScore(0, 10, 1.0); f2.setFooScore(1, 5, 2.0); - EXPECT_EQUAL(0, f2.getScore(10)); + EXPECT_EQ(0, f2.getScore(10)); } void @@ -88,21 +125,25 @@ expect_raw_score_calculated_on_the_fly(RankFixture& f) // For docids 9 and 10 the raw score is calculated on the fly // using a distance calculator over the attribute and query tensors. - EXPECT_EQUAL(1/(1+13.0), f.getScore(8)); - EXPECT_EQUAL(1/(1+(5.0-3.0)), f.getScore(9)); - EXPECT_EQUAL(1/(1+(7.0-3.0)), f.getScore(10)); + EXPECT_EQ(1/(1+13.0), f.getScore(8)); + EXPECT_EQ(1/(1+(5.0-3.0)), f.getScore(9)); + EXPECT_EQ(1/(1+(7.0-3.0)), f.getScore(10)); } -TEST_FF("raw score is calculated on the fly (using field setup)", - NoLabel(), RankFixture(0, 1, f1, fieldFeatureName, "tensor(x[2]):[3,11]")) +TEST(NnsClosenessTest, raw_score_is_calculated_on_the_fly_using_field_setup) { + NoLabel f1; + RankFixture f2(0, 1, f1, fieldFeatureName, "tensor(x[2]):[3,11]"); + ASSERT_FALSE(f2.failed()); expect_raw_score_calculated_on_the_fly(f2); } -TEST_FF("raw score is calculated on the fly (using label setup)", - SingleLabel("nns", 1), RankFixture(0, 1, f1, labelFeatureName, "tensor(x[2]):[3,11]")) +TEST(NnsClosenessTest, raw_score_is_calculated_on_the_fly_using_label_setup) { + SingleLabel f1("nns", 1); + RankFixture f2(0, 1, f1, labelFeatureName, "tensor(x[2]):[3,11]"); + ASSERT_FALSE(f2.failed()); expect_raw_score_calculated_on_the_fly(f2); } -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/features/nns_distance/nns_distance_test.cpp b/searchlib/src/tests/features/nns_distance/nns_distance_test.cpp index fff4c9f1c0e..acc67803886 100644 --- a/searchlib/src/tests/features/nns_distance/nns_distance_test.cpp +++ b/searchlib/src/tests/features/nns_distance/nns_distance_test.cpp @@ -6,7 +6,7 @@ #include <vespa/searchlib/fef/test/labels.h> #include <vespa/searchlib/test/features/distance_closeness_fixture.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/stringfmt.h> using search::feature_t; @@ -22,66 +22,106 @@ const vespalib::string fieldFeatureName("distance(bar)"); using RankFixture = DistanceClosenessFixture; -TEST_F("require that blueprint can be created from factory", BlueprintFactoryFixture) { +TEST(NnsDistanceTest, require_that_blueprint_can_be_created_from_factory) +{ + BlueprintFactoryFixture f; Blueprint::SP bp = f.factory.createBlueprint("distance"); EXPECT_TRUE(bp.get() != 0); EXPECT_TRUE(dynamic_cast<DistanceBlueprint*>(bp.get()) != 0); } -TEST_FFF("require that no features are dumped", DistanceBlueprint, IndexEnvironmentFixture, FeatureDumpFixture) { +TEST(NnsDistanceTest, require_that_no_features_are_dumped) +{ + DistanceBlueprint f1; + IndexEnvironmentFixture f2; + FeatureDumpFixture f3; f1.visitDumpFeatures(f2.indexEnv, f3); } -TEST_FF("require that setup can be done on random label", DistanceBlueprint, IndexEnvironmentFixture) { +TEST(NnsDistanceTest, require_that_setup_can_be_done_on_random_label) +{ + DistanceBlueprint f1; + IndexEnvironmentFixture f2; DummyDependencyHandler deps(f1); f1.setName(vespalib::make_string("%s(label,random_label)", f1.getBaseName().c_str())); EXPECT_TRUE(static_cast<Blueprint&>(f1).setup(f2.indexEnv, std::vector<vespalib::string>{"label", "random_label"})); } -TEST_FF("require that setup with unknown field fails", DistanceBlueprint, IndexEnvironmentFixture) { +TEST(NnsDistanceTest, require_that_setup_with_unknown_field_fails) +{ + DistanceBlueprint f1; + IndexEnvironmentFixture f2; DummyDependencyHandler deps(f1); f1.setName(vespalib::make_string("%s(field,random_fieldname)", f1.getBaseName().c_str())); EXPECT_FALSE(static_cast<Blueprint&>(f1).setup(f2.indexEnv, std::vector<vespalib::string>{"field", "random_fieldname"})); } -TEST_FF("require that no label gives max-double distance", NoLabel(), RankFixture(2, 2, f1, labelFeatureName)) { - EXPECT_EQUAL(std::numeric_limits<feature_t>::max(), f2.getScore(10)); +TEST(NnsDistanceTest, require_that_no_label_gives_max_double_distance) +{ + NoLabel f1; + RankFixture f2(2, 2, f1, labelFeatureName); + ASSERT_FALSE(f2.failed()); + EXPECT_EQ(std::numeric_limits<feature_t>::max(), f2.getScore(10)); } -TEST_FF("require that unrelated label gives max-double distance", SingleLabel("unrelated", 1), RankFixture(2, 2, f1, labelFeatureName)) { - EXPECT_EQUAL(std::numeric_limits<feature_t>::max(), f2.getScore(10)); +TEST(NnsDistanceTest, require_that_unrelated_label_gives_max_double_distance) +{ + SingleLabel f1("unrelated", 1); + RankFixture f2(2, 2, f1, labelFeatureName); + ASSERT_FALSE(f2.failed()); + EXPECT_EQ(std::numeric_limits<feature_t>::max(), f2.getScore(10)); } -TEST_FF("require that labeled item raw score can be obtained", SingleLabel("nns", 1), RankFixture(2, 2, f1, labelFeatureName)) { +TEST(NnsDistanceTest, require_that_labeled_item_raw_score_can_be_obtained) +{ + SingleLabel f1("nns", 1); + RankFixture f2(2, 2, f1, labelFeatureName); + ASSERT_FALSE(f2.failed()); f2.setFooScore(0, 10, 5.0); - EXPECT_EQUAL(5.0, f2.getScore(10)); + EXPECT_EQ(5.0, f2.getScore(10)); } -TEST_FF("require that field raw score can be obtained", NoLabel(), RankFixture(2, 2, f1, fieldFeatureName)) { +TEST(NnsDistanceTest, require_that_field_raw_score_can_be_obtained) +{ + NoLabel f1; + RankFixture f2(2, 2, f1, fieldFeatureName); + ASSERT_FALSE(f2.failed()); f2.setBarScore(0, 10, 5.0); - EXPECT_EQUAL(5.0, f2.getScore(10)); + EXPECT_EQ(5.0, f2.getScore(10)); } -TEST_FF("require that other raw scores are ignored", SingleLabel("nns", 2), RankFixture(2, 2, f1, labelFeatureName)) { +TEST(NnsDistanceTest, require_that_other_raw_scores_are_ignored) +{ + SingleLabel f1("nns", 2); + RankFixture f2(2, 2, f1, labelFeatureName); + ASSERT_FALSE(f2.failed()); f2.setFooScore(0, 10, 1.0); f2.setFooScore(1, 10, 2.0); f2.setBarScore(0, 10, 5.0); f2.setBarScore(1, 10, 6.0); - EXPECT_EQUAL(2.0, f2.getScore(10)); + EXPECT_EQ(2.0, f2.getScore(10)); } -TEST_FF("require that the correct raw score is used", NoLabel(), RankFixture(2, 2, f1, fieldFeatureName)) { +TEST(NnsDistanceTest, require_that_the_correct_raw_score_is_used) +{ + NoLabel f1; + RankFixture f2(2, 2, f1, fieldFeatureName); + ASSERT_FALSE(f2.failed()); f2.setFooScore(0, 10, 3.0); f2.setFooScore(1, 10, 4.0); f2.setBarScore(0, 10, 8.0); f2.setBarScore(1, 10, 7.0); - EXPECT_EQUAL(7.0, f2.getScore(10)); + EXPECT_EQ(7.0, f2.getScore(10)); } -TEST_FF("require that stale data is ignored", SingleLabel("nns", 2), RankFixture(2, 2, f1, labelFeatureName)) { +TEST(NnsDistanceTest, require_that_stale_data_is_ignored) +{ + SingleLabel f1("nns", 2); + RankFixture f2(2, 2, f1, labelFeatureName); + ASSERT_FALSE(f2.failed()); f2.setFooScore(0, 10, 1.0); f2.setFooScore(1, 5, 2.0); - EXPECT_EQUAL(std::numeric_limits<feature_t>::max(), f2.getScore(10)); + EXPECT_EQ(std::numeric_limits<feature_t>::max(), f2.getScore(10)); } void @@ -93,21 +133,25 @@ expect_raw_score_calculated_on_the_fly(RankFixture& f) // For docids 9 and 10 the raw score is calculated on the fly // using a distance calculator over the attribute and query tensors. - EXPECT_EQUAL(13.0, f.getScore(8)); - EXPECT_EQUAL((5-3), f.getScore(9)); - EXPECT_EQUAL((7-3), f.getScore(10)); + EXPECT_EQ(13.0, f.getScore(8)); + EXPECT_EQ((5-3), f.getScore(9)); + EXPECT_EQ((7-3), f.getScore(10)); } -TEST_FF("raw score is calculated on the fly (using field setup)", - NoLabel(), RankFixture(0, 1, f1, fieldFeatureName, "tensor(x[2]):[3,11]")) +TEST(NnsDistanceTest, raw_score_is_calculated_on_the_fly_using_field_setup) { + NoLabel f1; + RankFixture f2(0, 1, f1, fieldFeatureName, "tensor(x[2]):[3,11]"); + ASSERT_FALSE(f2.failed()); expect_raw_score_calculated_on_the_fly(f2); } -TEST_FF("raw score is calculated on the fly (using label setup)", - SingleLabel("nns", 1), RankFixture(0, 1, f1, labelFeatureName, "tensor(x[2]):[3,11]")) +TEST(NnsDistanceTest, raw_score_is_calculated_on_the_fly_using_label_setup) { + SingleLabel f1("nns", 1); + RankFixture f2(0, 1, f1, labelFeatureName, "tensor(x[2]):[3,11]"); + ASSERT_FALSE(f2.failed()); expect_raw_score_calculated_on_the_fly(f2); } -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/features/tensor/tensor_test.cpp b/searchlib/src/tests/features/tensor/tensor_test.cpp index 54807273aea..5ad30c61c37 100644 --- a/searchlib/src/tests/features/tensor/tensor_test.cpp +++ b/searchlib/src/tests/features/tensor/tensor_test.cpp @@ -121,7 +121,7 @@ struct ExecFixture .add({{"x", "b"}}, 5) .add({{"x", "c"}}, 7)); tensorAttr->setTensor(1, *doc_tensor); - directAttr->set_tensor(1, std::move(doc_tensor)); + directAttr->setTensor(1, *doc_tensor); for (const auto &attr : attrs) { attr->commit(); diff --git a/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp b/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp index 564824031a6..f4dda88b6f0 100644 --- a/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp +++ b/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp @@ -3,6 +3,7 @@ #include <vespa/searchlib/memoryindex/feature_store.h> #include <vespa/searchcommon/common/schema.h> #include <vespa/vespalib/gtest/gtest.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP("feature_store_test"); diff --git a/searchlib/src/tests/postinglistbm/stress_runner.cpp b/searchlib/src/tests/postinglistbm/stress_runner.cpp index 179e4f49ef4..3e3db3701c7 100644 --- a/searchlib/src/tests/postinglistbm/stress_runner.cpp +++ b/searchlib/src/tests/postinglistbm/stress_runner.cpp @@ -2,7 +2,6 @@ #include "stress_runner.h" -#include <vespa/fastos/thread.h> #include <vespa/searchlib/test/fakedata/fake_match_loop.h> #include <vespa/searchlib/test/fakedata/fakeposting.h> #include <vespa/searchlib/test/fakedata/fakeword.h> @@ -13,7 +12,7 @@ #include <condition_variable> #include <mutex> #include <vector> -#include <thread> +#include <vespa/vespalib/util/thread.h> #include <vespa/log/log.h> LOG_SETUP(".stress_runner"); @@ -43,7 +42,7 @@ private: uint32_t _stride; bool _unpack; - FastOS_ThreadPool *_threadPool; + vespalib::ThreadPool _threadPool; std::vector<StressWorkerUP> _workers; uint32_t _workersDone; @@ -88,7 +87,7 @@ public: double runWorkers(const std::string &postingFormat); }; -class StressWorker : public FastOS_Runnable { +class StressWorker : vespalib::Runnable { protected: StressMaster& _master; uint32_t _id; @@ -102,7 +101,7 @@ public: StressWorker(StressMaster& master, uint32_t id); virtual ~StressWorker(); - virtual void Run(FastOS_ThreadInterface* thisThread, void* arg) override; + virtual void run() override; }; class DirectStressWorker : public StressWorker { @@ -147,7 +146,7 @@ StressMaster::StressMaster(vespalib::Rand48 &rnd, _skipCommonPairsRate(skipCommonPairsRate), _stride(stride), _unpack(unpack), - _threadPool(nullptr), + _threadPool(), _workers(), _workersDone(0), _wordSet(wordSet), @@ -159,17 +158,12 @@ StressMaster::StressMaster(vespalib::Rand48 &rnd, _tasks() { LOG(info, "StressMaster::StressMaster()"); - - _threadPool = new FastOS_ThreadPool(400); } StressMaster::~StressMaster() { LOG(info, "StressMaster::~StressMaster()"); - - _threadPool->Close(); - delete _threadPool; - _threadPool = nullptr; + _threadPool.join(); _workers.clear(); dropPostings(); } @@ -329,7 +323,7 @@ StressMaster::runWorkers(const std::string &postingFormat) } for (auto& worker : _workers) { - _threadPool->NewThread(worker.get()); + _threadPool.start([obj = worker.get()](){obj->run();}); } { @@ -357,10 +351,8 @@ StressWorker::StressWorker(StressMaster& master, uint32_t id) StressWorker::~StressWorker() = default; void -StressWorker::Run(FastOS_ThreadInterface* thisThread, void* arg) +StressWorker::run() { - (void) thisThread; - (void) arg; LOG(debug, "StressWorker::Run(), id=%u", _id); bool unpack = _master.getUnpack(); diff --git a/searchlib/src/tests/sortspec/multilevelsort.cpp b/searchlib/src/tests/sortspec/multilevelsort.cpp index ec14f0c97e1..001903ff302 100644 --- a/searchlib/src/tests/sortspec/multilevelsort.cpp +++ b/searchlib/src/tests/sortspec/multilevelsort.cpp @@ -11,6 +11,7 @@ #include <vespa/vespalib/util/testclock.h> #include <vespa/vespalib/testkit/testapp.h> #include <type_traits> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP("multilevelsort_test"); diff --git a/searchlib/src/tests/tensor/distance_calculator/distance_calculator_test.cpp b/searchlib/src/tests/tensor/distance_calculator/distance_calculator_test.cpp index 5e556979254..ef4292ddbb4 100644 --- a/searchlib/src/tests/tensor/distance_calculator/distance_calculator_test.cpp +++ b/searchlib/src/tests/tensor/distance_calculator/distance_calculator_test.cpp @@ -18,6 +18,8 @@ using namespace vespalib::eval; using search::AttributeVector; +using OptSubspace = std::optional<uint32_t>; + std::unique_ptr<Value> make_tensor(const vespalib::string& expr) { return SimpleValue::from_spec(TensorSpec::from_expr(expr)); } @@ -49,6 +51,11 @@ public: auto calc = DistanceCalculator::make_with_validation(*attr, *qt); return calc->calc_raw_score(docid); } + OptSubspace calc_closest_subspace(uint32_t docid, const vespalib::string& query_tensor) { + auto qt = make_tensor(query_tensor); + auto calc = DistanceCalculator::make_with_validation(*attr, *qt); + return calc->calc_closest_subspace(attr->asTensorAttribute()->get_vectors(docid)); + } void make_calc_throws(const vespalib::string& query_tensor) { auto qt = make_tensor(query_tensor); DistanceCalculator::make_with_validation(*attr, *qt); @@ -63,9 +70,11 @@ TEST_F(DistanceCalculatorTest, calculation_over_dense_tensor_attribute) vespalib::string qt = "tensor(y[2]):[7,10]"; EXPECT_DOUBLE_EQ(16, calc_distance(1, qt)); EXPECT_DOUBLE_EQ(max_distance, calc_distance(2, qt)); + EXPECT_EQ(OptSubspace(0), calc_closest_subspace(1, qt)); EXPECT_DOUBLE_EQ(1.0/(1.0 + 4.0), calc_rawscore(1, qt)); EXPECT_DOUBLE_EQ(0.0, calc_rawscore(2, qt)); + EXPECT_EQ(OptSubspace(), calc_closest_subspace(2, qt)); } TEST_F(DistanceCalculatorTest, calculation_over_mixed_tensor_attribute) @@ -77,8 +86,12 @@ TEST_F(DistanceCalculatorTest, calculation_over_mixed_tensor_attribute) vespalib::string qt_2 = "tensor(y[2]):[1,10]"; EXPECT_DOUBLE_EQ(16, calc_distance(1, qt_1)); EXPECT_DOUBLE_EQ(4, calc_distance(1, qt_2)); + EXPECT_EQ(OptSubspace(1), calc_closest_subspace(1, qt_1)); + EXPECT_EQ(OptSubspace(0), calc_closest_subspace(1, qt_2)); EXPECT_DOUBLE_EQ(max_distance, calc_distance(2, qt_1)); EXPECT_DOUBLE_EQ(max_distance, calc_distance(3, qt_1)); + EXPECT_EQ(OptSubspace(), calc_closest_subspace(2, qt_1)); + EXPECT_EQ(OptSubspace(), calc_closest_subspace(3, qt_1)); EXPECT_DOUBLE_EQ(1.0/(1.0 + 4.0), calc_rawscore(1, qt_1)); EXPECT_DOUBLE_EQ(1.0/(1.0 + 2.0), calc_rawscore(1, qt_2)); diff --git a/searchlib/src/tests/transactionlog/translogclient_test.cpp b/searchlib/src/tests/transactionlog/translogclient_test.cpp index a1a42b592b2..af277ecbc68 100644 --- a/searchlib/src/tests/transactionlog/translogclient_test.cpp +++ b/searchlib/src/tests/transactionlog/translogclient_test.cpp @@ -11,7 +11,6 @@ #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/fnet/transport.h> #include <vespa/fastos/file.h> -#include <vespa/fastos/thread.h> #include <thread> #include <vespa/log/log.h> @@ -480,16 +479,14 @@ getMaxSessionRunTime(TransLogServer &tls, const vespalib::string &domain) } struct TLS { - FastOS_ThreadPool threadPool; FNET_Transport transport; TransLogServer tls; TLS(const vespalib::string &name, int listenPort, const vespalib::string &baseDir, const common::FileHeaderContext &fileHeaderContext, const DomainConfig & cfg, size_t maxThreads = 4) - : threadPool(), - transport(), + : transport(), tls(transport, name, listenPort, baseDir, fileHeaderContext, cfg, maxThreads) { - transport.Start(&threadPool); + transport.Start(); } ~TLS() { transport.ShutDown(true); diff --git a/searchlib/src/tests/transactionlogstress/translogstress.cpp b/searchlib/src/tests/transactionlogstress/translogstress.cpp index eb457f312e6..124eb39e84b 100644 --- a/searchlib/src/tests/transactionlogstress/translogstress.cpp +++ b/searchlib/src/tests/transactionlogstress/translogstress.cpp @@ -5,7 +5,6 @@ #include <vespa/searchlib/transactionlog/translogclient.h> #include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/searchlib/util/runnable.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/fnet/transport.h> #include <vespa/vespalib/util/signalhandler.h> @@ -20,7 +19,6 @@ LOG_SETUP("translogstress"); using vespalib::nbostream; -using search::Runnable; using std::shared_ptr; using vespalib::make_string; using vespalib::ConstBufferRef; @@ -190,7 +188,7 @@ public: //----------------------------------------------------------------------------- // FeederThread //----------------------------------------------------------------------------- -class FeederThread : public Runnable +class FeederThread { private: std::string _tlsSpec; @@ -203,6 +201,8 @@ private: SerialNum _current; SerialNum _lastCommited; vespalib::Timer _timer; + std::atomic<bool> _done; + std::thread _thread; void commitPacket(); bool addEntry(const Packet::Entry & e); @@ -210,8 +210,11 @@ private: public: FeederThread(FNET_Transport & transport, const std::string & tlsSpec, const std::string & domain, const EntryGenerator & generator, uint32_t feedRate, size_t packetSize); - ~FeederThread() override; - void doRun() override; + ~FeederThread(); + void doRun(); + void start() { _thread = std::thread([this](){doRun();}); } + void stop() { _done = true; } + void join() { _thread.join(); } SerialNumRange getRange() const { return SerialNumRange(1, _lastCommited); } }; @@ -450,7 +453,7 @@ VisitorAgent::receive(const Packet & packet) //----------------------------------------------------------------------------- // ControllerThread //----------------------------------------------------------------------------- -class ControllerThread : public Runnable +class ControllerThread { private: std::string _tlsSpec; @@ -466,7 +469,9 @@ private: SerialNum _begin; SerialNum _end; size_t _count; - + std::atomic<bool> _done; + std::thread _thread; + void getStatus(); void makeRandomVisitorVector(); @@ -475,8 +480,10 @@ public: uint32_t numVisitors, vespalib::duration visitorInterval, vespalib::duration pruneInterval); ~ControllerThread(); std::vector<std::shared_ptr<VisitorAgent> > & getVisitors() { return _visitors; } - virtual void doRun() override; - + void doRun(); + void start() { _thread = std::thread([this](){doRun();}); } + void stop() { _done = true; } + void join() { _thread.join(); } }; ControllerThread::ControllerThread(FNET_Transport & transport, const std::string & tlsSpec, const std::string & domain, @@ -698,7 +705,6 @@ TransLogStress::main(int argc, char **argv) } // start transaction log server - FastOS_ThreadPool threadPool; FNET_Transport transport; DummyFileHeaderContext fileHeaderContext; TransLogServer tls(transport, "server", 17897, ".", fileHeaderContext, DomainConfig().setPartSizeLimit(_cfg.domainPartSize)); @@ -719,12 +725,12 @@ TransLogStress::main(int argc, char **argv) // start feeder and controller FeederThread feeder(transport, tlsSpec, domain, generator, _cfg.feedRate, _cfg.packetSize); - threadPool.NewThread(&feeder); + feeder.start(); std::this_thread::sleep_for(sleepTime); ControllerThread controller(transport, tlsSpec, domain, generator, _cfg.numVisitors, _cfg.visitorInterval, _cfg.pruneInterval); - threadPool.NewThread(&controller); + controller.start(); // stop feeder and controller std::this_thread::sleep_for(_cfg.stressTime); @@ -751,8 +757,6 @@ TransLogStress::main(int argc, char **argv) std::cout << "</visitor>" << std::endl; } - threadPool.Close(); - return 0; } diff --git a/searchlib/src/vespa/searchcommon/attribute/basictype.cpp b/searchlib/src/vespa/searchcommon/attribute/basictype.cpp index d0d90d1c9d5..41221457400 100644 --- a/searchlib/src/vespa/searchcommon/attribute/basictype.cpp +++ b/searchlib/src/vespa/searchcommon/attribute/basictype.cpp @@ -19,7 +19,8 @@ const BasicType::TypeInfo BasicType::_typeTable[BasicType::MAX_TYPE] = { { BasicType::DOUBLE, sizeof(double), "double" }, { BasicType::PREDICATE, 0, "predicate" }, { BasicType::TENSOR, 0, "tensor" }, - { BasicType::REFERENCE, 12, "reference" } + { BasicType::REFERENCE, 12, "reference" }, + { BasicType::RAW, 0, "raw" } }; BasicType::Type diff --git a/searchlib/src/vespa/searchcommon/attribute/basictype.h b/searchlib/src/vespa/searchcommon/attribute/basictype.h index bd7b4a2b4bc..46387dd2738 100644 --- a/searchlib/src/vespa/searchcommon/attribute/basictype.h +++ b/searchlib/src/vespa/searchcommon/attribute/basictype.h @@ -24,6 +24,7 @@ class BasicType PREDICATE = 11, TENSOR = 12, REFERENCE = 13, + RAW = 14, MAX_TYPE }; diff --git a/searchlib/src/vespa/searchcommon/attribute/iattributevector.h b/searchlib/src/vespa/searchcommon/attribute/iattributevector.h index 837aead18fd..381dab9c844 100644 --- a/searchlib/src/vespa/searchcommon/attribute/iattributevector.h +++ b/searchlib/src/vespa/searchcommon/attribute/iattributevector.h @@ -128,16 +128,11 @@ public: virtual double getFloat(DocId doc) const = 0; /** - * Returns the first value stored for the given document as a string. - * Uses the given buffer to store the actual string if no underlying - * string storage is used for this attribute vector. + * Return raw value. * - * @param docId document identifier - * @param buffer content buffer to optionally store the string - * @param sz the size of the buffer - * @return the string value - **/ - virtual const char * getString(DocId doc, char * buffer, size_t sz) const = 0; + * TODO: Consider accessing via new IRawAttribute interface class. + */ + virtual vespalib::ConstArrayRef<char> get_raw(DocId doc) const = 0; /** * Returns the first value stored for the given document as an enum value. @@ -370,6 +365,11 @@ public: virtual bool isPredicateType() const { return getBasicType() == BasicType::PREDICATE; } virtual bool isTensorType() const { return getBasicType() == BasicType::TENSOR; } virtual bool isReferenceType() const { return getBasicType() == BasicType::REFERENCE; } + virtual bool is_raw_type() const noexcept { + BasicType::Type t = getBasicType(); + return t == BasicType::RAW || + t == BasicType::STRING; + } /** * Returns whether this is a multi value attribute. diff --git a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt index 72c7efe3094..6d9b8fbf3e0 100644 --- a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt @@ -25,6 +25,7 @@ vespa_add_library(searchlib_attribute OBJECT attrvector.cpp basename.cpp bitvector_search_cache.cpp + blob_sequence_reader.cpp changevector.cpp configconverter.cpp copy_multi_value_read_view.cpp @@ -104,6 +105,8 @@ vespa_add_library(searchlib_attribute OBJECT postinglisttraits.cpp postingstore.cpp predicate_attribute.cpp + raw_buffer_store.cpp + raw_buffer_type_mapper.cpp raw_multi_value_read_view.cpp readerbase.cpp reference_attribute.cpp @@ -123,6 +126,7 @@ vespa_add_library(searchlib_attribute OBJECT single_enum_search_context.cpp single_numeric_enum_search_context.cpp single_numeric_search_context.cpp + single_raw_attribute.cpp single_small_numeric_search_context.cpp single_string_enum_search_context.cpp single_string_enum_hint_search_context.cpp diff --git a/searchlib/src/vespa/searchlib/attribute/attrvector.h b/searchlib/src/vespa/searchlib/attribute/attrvector.h index 05dd611c187..0b71bdbcbcf 100644 --- a/searchlib/src/vespa/searchlib/attribute/attrvector.h +++ b/searchlib/src/vespa/searchlib/attribute/attrvector.h @@ -165,16 +165,13 @@ class StringDirectAttrVector : public search::StringDirectAttribute public: StringDirectAttrVector(const vespalib::string & baseFileName); StringDirectAttrVector(const vespalib::string & baseFileName, const Config & c); - const char * getString(DocId doc, char * v, size_t sz) const override { - (void) v; (void) sz; return getHelper(doc, 0); - } uint32_t get(DocId doc, const char ** v, uint32_t sz) const override { return getAllHelper(doc, v, sz); } + const char * get(DocId doc) const override { return getHelper(doc, 0); } private: uint32_t get(DocId doc, vespalib::string * v, uint32_t sz) const override { return getAllHelper(doc, v, sz); } uint32_t get(DocId doc, EnumHandle * e, uint32_t sz) const override { return getAllEnumHelper(doc, e, sz); } - const char * get(DocId doc) const override { return getHelper(doc, 0); } EnumHandle getEnum(DocId doc) const override { return getEnumHelper(doc, 0); } uint32_t getValueCount(DocId doc) const override { return getValueCountHelper(doc); } uint32_t get(DocId doc, WeightedEnum * e, uint32_t sz) const override { return getAllEnumHelper(doc, e, sz); } diff --git a/searchlib/src/vespa/searchlib/tensor/blob_sequence_reader.cpp b/searchlib/src/vespa/searchlib/attribute/blob_sequence_reader.cpp index 0d86af2f3a5..aca3a37492c 100644 --- a/searchlib/src/vespa/searchlib/tensor/blob_sequence_reader.cpp +++ b/searchlib/src/vespa/searchlib/attribute/blob_sequence_reader.cpp @@ -3,7 +3,7 @@ #include "blob_sequence_reader.h" #include <vespa/fastos/file.h> -namespace search::tensor { +namespace search::attribute { void BlobSequenceReader::readBlob(void *buf, size_t len) { diff --git a/searchlib/src/vespa/searchlib/tensor/blob_sequence_reader.h b/searchlib/src/vespa/searchlib/attribute/blob_sequence_reader.h index adad87d8cfe..26520a784ac 100644 --- a/searchlib/src/vespa/searchlib/tensor/blob_sequence_reader.h +++ b/searchlib/src/vespa/searchlib/attribute/blob_sequence_reader.h @@ -4,7 +4,7 @@ #include <vespa/searchlib/attribute/readerbase.h> -namespace search::tensor { +namespace search::attribute { /** * Utility for reading an attribute data file where diff --git a/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp b/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp index fe2b0c9f989..1a0d24b0595 100644 --- a/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp +++ b/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp @@ -7,6 +7,7 @@ #include "singlestringattribute.h" #include "singleboolattribute.h" #include "singlenumericattribute.hpp" +#include "single_raw_attribute.h" #include <vespa/eval/eval/fast_value.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/serialized_fast_value_attribute.h> @@ -51,6 +52,8 @@ AttributeFactory::createSingleStd(stringref name, const Config & info) } case BasicType::REFERENCE: return std::make_shared<attribute::ReferenceAttribute>(name, info); + case BasicType::RAW: + return std::make_shared<attribute::SingleRawAttribute>(name, info); default: break; } diff --git a/searchlib/src/vespa/searchlib/attribute/floatbase.cpp b/searchlib/src/vespa/searchlib/attribute/floatbase.cpp index b9d7fb7c81b..dad83eaa778 100644 --- a/searchlib/src/vespa/searchlib/attribute/floatbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/floatbase.cpp @@ -88,11 +88,10 @@ bool FloatingPointAttribute::apply(DocId doc, const ArithmeticValueUpdate & op) return retval; } -const char * -FloatingPointAttribute::getString(DocId doc, char * s, size_t sz) const { - double v = getFloat(doc); - snprintf(s, sz, "%g", v); - return s; +vespalib::ConstArrayRef<char> +FloatingPointAttribute::get_raw(DocId) const +{ + return {}; } vespalib::MemoryUsage diff --git a/searchlib/src/vespa/searchlib/attribute/floatbase.h b/searchlib/src/vespa/searchlib/attribute/floatbase.h index 288b66195e4..675ea50a4fc 100644 --- a/searchlib/src/vespa/searchlib/attribute/floatbase.h +++ b/searchlib/src/vespa/searchlib/attribute/floatbase.h @@ -30,7 +30,6 @@ public: bool applyWeight(DocId doc, const FieldValue& fv, const document::AssignValueUpdate& wAdjust) override; uint32_t clearDoc(DocId doc) override; protected: - const char * getString(DocId doc, char * s, size_t sz) const override; FloatingPointAttribute(const vespalib::string & name, const Config & c); using Change = ChangeTemplate<NumericChangeData<double>>; using ChangeVector = ChangeVectorT<Change>; @@ -38,6 +37,7 @@ protected: vespalib::MemoryUsage getChangeVectorMemoryUsage() const override; private: + vespalib::ConstArrayRef<char> get_raw(DocId) const override; uint32_t get(DocId doc, vespalib::string * v, uint32_t sz) const override; uint32_t get(DocId doc, const char ** v, uint32_t sz) const override; uint32_t get(DocId doc, WeightedString * v, uint32_t sz) const override; diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp index 489b2fb5e6e..a1a5e9f7894 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp @@ -51,8 +51,10 @@ double ImportedAttributeVectorReadGuard::getFloat(DocId doc) const { return _target_attribute.getFloat(getTargetLid(doc)); } -const char *ImportedAttributeVectorReadGuard::getString(DocId doc, char *buffer, size_t sz) const { - return _target_attribute.getString(getTargetLid(doc), buffer, sz); +vespalib::ConstArrayRef<char> +ImportedAttributeVectorReadGuard::get_raw(DocId doc) const +{ + return _target_attribute.get_raw(getTargetLid(doc)); } IAttributeVector::EnumHandle ImportedAttributeVectorReadGuard::getEnum(DocId doc) const { diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h index fd9856a032c..cb48399f688 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h @@ -42,7 +42,7 @@ public: uint32_t getMaxValueCount() const override; largeint_t getInt(DocId doc) const override; double getFloat(DocId doc) const override; - const char *getString(DocId doc, char *buffer, size_t sz) const override; + vespalib::ConstArrayRef<char> get_raw(DocId doc) const override; EnumHandle getEnum(DocId doc) const override; uint32_t get(DocId docId, largeint_t *buffer, uint32_t sz) const override; uint32_t get(DocId docId, double *buffer, uint32_t sz) const override; diff --git a/searchlib/src/vespa/searchlib/attribute/integerbase.cpp b/searchlib/src/vespa/searchlib/attribute/integerbase.cpp index b9d33f0aa9e..b6365412ae5 100644 --- a/searchlib/src/vespa/searchlib/attribute/integerbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/integerbase.cpp @@ -55,19 +55,13 @@ uint32_t IntegerAttribute::get(DocId doc, WeightedConstChar * v, uint32_t sz) co (void) sz; return 0; } -const char * -IntegerAttribute::getString(DocId doc, char * s, size_t sz) const { - if (sz > 1) { - largeint_t v = getInt(doc); - auto res = std::to_chars(s, s + sz - 1, v, 10); - if (res.ec == std::errc()) { - res.ptr[0] = 0; - } else { - s[0] = 0; - } - } - return s; + +vespalib::ConstArrayRef<char> +IntegerAttribute::get_raw(DocId) const +{ + return {}; } + uint32_t IntegerAttribute::get(DocId doc, vespalib::string * s, uint32_t sz) const { largeint_t * v = new largeint_t[sz]; diff --git a/searchlib/src/vespa/searchlib/attribute/integerbase.h b/searchlib/src/vespa/searchlib/attribute/integerbase.h index 7cb791e204e..3c137c280c2 100644 --- a/searchlib/src/vespa/searchlib/attribute/integerbase.h +++ b/searchlib/src/vespa/searchlib/attribute/integerbase.h @@ -36,7 +36,7 @@ protected: using ChangeVector = ChangeVectorT<Change>; ChangeVector _changes; private: - const char * getString(DocId doc, char * s, size_t sz) const override; + vespalib::ConstArrayRef<char> get_raw(DocId) const override; uint32_t get(DocId doc, vespalib::string * v, uint32_t sz) const override; uint32_t get(DocId doc, const char ** v, uint32_t sz) const override; uint32_t get(DocId doc, WeightedString * v, uint32_t sz) const override; diff --git a/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.cpp index d8b01afe094..c208cc6fbfa 100644 --- a/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.cpp @@ -43,8 +43,9 @@ NotImplementedAttribute::getFloat(DocId) const { notImplemented(); } -const char * -NotImplementedAttribute::getString(DocId, char *, size_t) const { +vespalib::ConstArrayRef<char> +NotImplementedAttribute::get_raw(DocId) const +{ notImplemented(); } diff --git a/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h b/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h index e824b0dd691..338ebb0f3ab 100644 --- a/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h +++ b/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h @@ -14,7 +14,7 @@ struct NotImplementedAttribute : AttributeVector { uint32_t getValueCount(DocId) const override; largeint_t getInt(DocId) const override; double getFloat(DocId) const override; - const char * getString(DocId, char *, size_t) const override; + vespalib::ConstArrayRef<char> get_raw(DocId) const override; uint32_t get(DocId, largeint_t *, uint32_t) const override; uint32_t get(DocId, double *, uint32_t) const override; uint32_t get(DocId, vespalib::string *, uint32_t) const override; diff --git a/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.cpp b/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.cpp new file mode 100644 index 00000000000..74894728ff4 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.cpp @@ -0,0 +1,69 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "raw_buffer_store.h" +#include <vespa/vespalib/datastore/array_store.hpp> +#include <cassert> + +using vespalib::alloc::MemoryAllocator; +using vespalib::datastore::EntryRef; + +namespace { + +constexpr float ALLOC_GROW_FACTOR = 0.2; + +} + +namespace search::attribute { + +RawBufferStore::RawBufferStore(std::shared_ptr<vespalib::alloc::MemoryAllocator> allocator, uint32_t max_small_buffer_type_id, double grow_factor) + : _array_store(ArrayStoreType::optimizedConfigForHugePage(max_small_buffer_type_id, + RawBufferTypeMapper(max_small_buffer_type_id, grow_factor), + MemoryAllocator::HUGEPAGE_SIZE, + MemoryAllocator::PAGE_SIZE, + 8_Ki, ALLOC_GROW_FACTOR), + std::move(allocator), RawBufferTypeMapper(max_small_buffer_type_id, grow_factor)) +{ +} + +RawBufferStore::~RawBufferStore() = default; + +vespalib::ConstArrayRef<char> +RawBufferStore::get(EntryRef ref) const +{ + auto array = _array_store.get(ref); + uint32_t size = 0; + assert(array.size() >= sizeof(size)); + memcpy(&size, array.data(), sizeof(size)); + assert(array.size() >= sizeof(size) + size); + return {array.data() + sizeof(size), size}; +} + +EntryRef +RawBufferStore::set(vespalib::ConstArrayRef<char> raw) +{ + uint32_t size = raw.size(); + if (size == 0) { + return EntryRef(); + } + size_t buffer_size = raw.size() + sizeof(size); + auto& mapper = _array_store.get_mapper(); + auto type_id = mapper.get_type_id(buffer_size); + auto array_size = (type_id != 0) ? mapper.get_array_size(type_id) : buffer_size; + assert(array_size >= buffer_size); + auto ref = _array_store.allocate(array_size); + auto buf = _array_store.get_writable(ref); + memcpy(buf.data(), &size, sizeof(size)); + memcpy(buf.data() + sizeof(size), raw.data(), size); + if (array_size > buffer_size) { + memset(buf.data() + buffer_size, 0, array_size - buffer_size); + } + return ref; +} + +std::unique_ptr<vespalib::datastore::ICompactionContext> +RawBufferStore::start_compact(const vespalib::datastore::CompactionStrategy& compaction_strategy) +{ + return _array_store.compact_worst(compaction_strategy); +} + +} diff --git a/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.h b/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.h new file mode 100644 index 00000000000..60132c70852 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.h @@ -0,0 +1,35 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/datastore/array_store.h> +#include "raw_buffer_type_mapper.h" + +namespace search::attribute { + +/** + * Class handling storage of raw values in an array store. A stored entry + * starts with 4 bytes that contains the size of the raw value. + */ +class RawBufferStore +{ + using EntryRef = vespalib::datastore::EntryRef; + using RefType = vespalib::datastore::EntryRefT<19>; + using ArrayStoreType = vespalib::datastore::ArrayStore<char, RefType, RawBufferTypeMapper>; + using generation_t = vespalib::GenerationHandler::generation_t; + + ArrayStoreType _array_store; +public: + RawBufferStore(std::shared_ptr<vespalib::alloc::MemoryAllocator> allocator, uint32_t max_small_buffer_type_id, double grow_factor); + ~RawBufferStore(); + EntryRef set(vespalib::ConstArrayRef<char> raw); + vespalib::ConstArrayRef<char> get(EntryRef ref) const; + void remove(EntryRef ref) { _array_store.remove(ref); } + vespalib::MemoryUsage update_stat(const vespalib::datastore::CompactionStrategy& compaction_strategy) { return _array_store.update_stat(compaction_strategy); } + bool consider_compact() const noexcept { return _array_store.consider_compact(); } + std::unique_ptr<vespalib::datastore::ICompactionContext> start_compact(const vespalib::datastore::CompactionStrategy& compaction_strategy); + void reclaim_memory(generation_t oldest_used_gen) { _array_store.reclaim_memory(oldest_used_gen); } + void assign_generation(generation_t current_gen) { _array_store.assign_generation(current_gen); } +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/raw_buffer_type_mapper.cpp b/searchlib/src/vespa/searchlib/attribute/raw_buffer_type_mapper.cpp new file mode 100644 index 00000000000..29245fb403a --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/raw_buffer_type_mapper.cpp @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "raw_buffer_type_mapper.h" +#include <vespa/vespalib/datastore/aligner.h> +#include <algorithm> +#include <cmath> +#include <limits> + +using vespalib::datastore::Aligner; +using vespalib::datastore::ArrayStoreTypeMapper; + +namespace search::attribute { + +RawBufferTypeMapper::RawBufferTypeMapper() + : ArrayStoreTypeMapper() +{ +} + +RawBufferTypeMapper::RawBufferTypeMapper(uint32_t max_small_buffer_type_id, double grow_factor) + : ArrayStoreTypeMapper() +{ + Aligner<4> aligner; + _array_sizes.reserve(max_small_buffer_type_id + 1); + _array_sizes.emplace_back(0); // type id 0 uses LargeArrayBufferType<char> + size_t array_size = 8u; + for (uint32_t type_id = 1; type_id <= max_small_buffer_type_id; ++type_id) { + if (type_id > 1) { + array_size = std::max(array_size + 4, static_cast<size_t>(std::floor(array_size * grow_factor))); + array_size = aligner.align(array_size); + } + if (array_size > std::numeric_limits<uint32_t>::max()) { + break; + } + _array_sizes.emplace_back(array_size); + } +} + +RawBufferTypeMapper::~RawBufferTypeMapper() = default; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/raw_buffer_type_mapper.h b/searchlib/src/vespa/searchlib/attribute/raw_buffer_type_mapper.h new file mode 100644 index 00000000000..88c213c8979 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/raw_buffer_type_mapper.h @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/datastore/array_store_type_mapper.h> + +namespace vespalib::datastore { + +template <typename EntryT> class SmallArrayBufferType; +template <typename EntryT> class LargeArrayBufferType; + +} + +namespace search::attribute { + +/* + * This class provides mapping between type ids and array sizes needed for + * storing a raw value. + */ +class RawBufferTypeMapper : public vespalib::datastore::ArrayStoreTypeMapper +{ +public: + using SmallBufferType = vespalib::datastore::SmallArrayBufferType<char>; + using LargeBufferType = vespalib::datastore::LargeArrayBufferType<char>; + + RawBufferTypeMapper(); + RawBufferTypeMapper(uint32_t max_small_buffer_type_id, double grow_factor); + ~RawBufferTypeMapper(); +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.cpp new file mode 100644 index 00000000000..9746929c666 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.cpp @@ -0,0 +1,174 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "single_raw_attribute.h" +#include <vespa/searchcommon/attribute/config.h> +#include <vespa/vespalib/datastore/array_store.hpp> + +using vespalib::alloc::MemoryAllocator; +using vespalib::datastore::EntryRef; + +namespace { + +constexpr double mapper_grow_factor = 1.03; + +constexpr uint32_t max_small_buffer_type_id = 500u; + +} + +namespace search::attribute { + +SingleRawAttribute::SingleRawAttribute(const vespalib::string& name, const Config& config) + : NotImplementedAttribute(name, config), + _ref_vector(config.getGrowStrategy(), getGenerationHolder()), + _raw_store(get_memory_allocator(), max_small_buffer_type_id, mapper_grow_factor) +{ +} + +SingleRawAttribute::~SingleRawAttribute() +{ + getGenerationHolder().reclaim_all(); +} + +void +SingleRawAttribute::reclaim_memory(generation_t oldest_used_gen) +{ + _raw_store.reclaim_memory(oldest_used_gen); + getGenerationHolder().reclaim(oldest_used_gen); +} + +void +SingleRawAttribute::before_inc_generation(generation_t current_gen) +{ + getGenerationHolder().assign_generation(current_gen); + _raw_store.assign_generation(current_gen); +} + +bool +SingleRawAttribute::addDoc(DocId &docId) +{ + bool incGen = _ref_vector.isFull(); + _ref_vector.push_back(AtomicEntryRef()); + AttributeVector::incNumDocs(); + docId = AttributeVector::getNumDocs() - 1; + updateUncommittedDocIdLimit(docId); + if (incGen) { + incGeneration(); + } else { + reclaim_unused_memory(); + } + return true; +} + +void +SingleRawAttribute::onCommit() +{ + incGeneration(); + if (_raw_store.consider_compact()) { + auto context = _raw_store.start_compact(getConfig().getCompactionStrategy()); + if (context) { + context->compact(vespalib::ArrayRef<AtomicEntryRef>(&_ref_vector[0], _ref_vector.size())); + } + incGeneration(); + updateStat(true); + } +} + +void +SingleRawAttribute::onUpdateStat() +{ + vespalib::MemoryUsage total = update_stat(); + this->updateStatistics(_ref_vector.size(), + _ref_vector.size(), + total.allocatedBytes(), + total.usedBytes(), + total.deadBytes(), + total.allocatedBytesOnHold()); +} + +vespalib::MemoryUsage +SingleRawAttribute::update_stat() +{ + vespalib::MemoryUsage result = _ref_vector.getMemoryUsage(); + result.merge(_raw_store.update_stat(getConfig().getCompactionStrategy())); + result.mergeGenerationHeldBytes(getGenerationHolder().get_held_bytes()); + return result; +} + +vespalib::ConstArrayRef<char> +SingleRawAttribute::get_raw(DocId docid) const +{ + EntryRef ref; + if (docid < getCommittedDocIdLimit()) { + ref = acquire_entry_ref(docid); + } + if (!ref.valid()) { + return {}; + } + return _raw_store.get(ref); +} + +void +SingleRawAttribute::set_raw(DocId docid, vespalib::ConstArrayRef<char> raw) +{ + auto ref = _raw_store.set(raw); + assert(docid < _ref_vector.size()); + updateUncommittedDocIdLimit(docid); + auto& elem_ref = _ref_vector[docid]; + EntryRef old_ref(elem_ref.load_relaxed()); + elem_ref.store_release(ref); + if (old_ref.valid()) { + _raw_store.remove(old_ref); + } +} + +uint32_t +SingleRawAttribute::clearDoc(DocId docId) +{ + updateUncommittedDocIdLimit(docId); + auto& elem_ref = _ref_vector[docId]; + EntryRef old_ref(elem_ref.load_relaxed()); + elem_ref.store_relaxed(EntryRef()); + if (old_ref.valid()) { + _raw_store.remove(old_ref); + return 1u; + } + return 0u; +} + +long +SingleRawAttribute::onSerializeForAscendingSort(DocId doc, void * serTo, long available, const common::BlobConverter * bc) const +{ + auto raw = get_raw(doc); + vespalib::ConstBufferRef buf(raw.data(), raw.size()); + if (bc != nullptr) { + buf = bc->convert(buf); + } + if (available >= (long)buf.size()) { + memcpy(serTo, buf.data(), buf.size()); + } else { + return -1; + } + return buf.size(); +} + +long +SingleRawAttribute::onSerializeForDescendingSort(DocId doc, void * serTo, long available, const common::BlobConverter * bc) const +{ + auto raw = get_raw(doc); + vespalib::ConstBufferRef buf(raw.data(), raw.size()); + if (bc != nullptr) { + buf = bc->convert(buf); + } + if (available >= (long)buf.size()) { + auto *dst = static_cast<unsigned char *>(serTo); + const auto * src(static_cast<const uint8_t *>(buf.data())); + for (size_t i(0); i < buf.size(); ++i) { + dst[i] = 0xff - src[i]; + } + } else { + return -1; + } + return buf.size(); +} + +} diff --git a/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.h b/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.h new file mode 100644 index 00000000000..d7ea321a3d4 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.h @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "not_implemented_attribute.h" +#include "raw_buffer_store.h" +#include <vespa/vespalib/util/rcuvector.h> + +namespace search::attribute { + +/** + * Attribute vector storing a single raw value per document. + */ +class SingleRawAttribute : public NotImplementedAttribute +{ + using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; + using EntryRef = vespalib::datastore::EntryRef; + using RefVector = vespalib::RcuVectorBase<AtomicEntryRef>; + + RefVector _ref_vector; + RawBufferStore _raw_store; + + vespalib::MemoryUsage update_stat(); + EntryRef acquire_entry_ref(DocId docid) const noexcept { return _ref_vector.acquire_elem_ref(docid).load_acquire(); } +public: + SingleRawAttribute(const vespalib::string& name, const Config& config); + ~SingleRawAttribute() override; + void onCommit() override; + void onUpdateStat() override; + void reclaim_memory(generation_t oldest_used_gen) override; + void before_inc_generation(generation_t current_gen) override; + bool addDoc(DocId &docId) override; + vespalib::ConstArrayRef<char> get_raw(DocId docid) const override; + void set_raw(DocId docid, vespalib::ConstArrayRef<char> raw); + uint32_t clearDoc(DocId docId) override; + long onSerializeForAscendingSort(DocId, void *, long, const common::BlobConverter *) const override; + long onSerializeForDescendingSort(DocId, void *, long, const common::BlobConverter *) const override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp index 3a9f88babfe..2800d7c3f6d 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp @@ -85,6 +85,13 @@ StringAttribute::getFloat(DocId doc) const { return vespalib::locale::c::strtod(get(doc), nullptr); } +vespalib::ConstArrayRef<char> +StringAttribute::get_raw(DocId doc) const +{ + const char * s = get(doc); + return {s, s ? ::strlen(s) : 0u}; +} + uint32_t StringAttribute::get(DocId doc, double * v, uint32_t sz) const { @@ -112,7 +119,6 @@ StringAttribute::get(DocId doc, largeint_t * v, uint32_t sz) const long StringAttribute::onSerializeForAscendingSort(DocId doc, void * serTo, long available, const common::BlobConverter * bc) const { - auto *dst = static_cast<unsigned char *>(serTo); const char *value(get(doc)); int size = strlen(value) + 1; vespalib::ConstBufferRef buf(value, size); @@ -120,7 +126,7 @@ StringAttribute::onSerializeForAscendingSort(DocId doc, void * serTo, long avail buf = bc->convert(buf); } if (available >= (long)buf.size()) { - memcpy(dst, buf.data(), buf.size()); + memcpy(serTo, buf.data(), buf.size()); } else { return -1; } @@ -130,8 +136,6 @@ StringAttribute::onSerializeForAscendingSort(DocId doc, void * serTo, long avail long StringAttribute::onSerializeForDescendingSort(DocId doc, void * serTo, long available, const common::BlobConverter * bc) const { - (void) bc; - auto *dst = static_cast<unsigned char *>(serTo); const char *value(get(doc)); int size = strlen(value) + 1; vespalib::ConstBufferRef buf(value, size); @@ -139,6 +143,7 @@ StringAttribute::onSerializeForDescendingSort(DocId doc, void * serTo, long avai buf = bc->convert(buf); } if (available >= (long)buf.size()) { + auto *dst = static_cast<unsigned char *>(serTo); const auto * src(static_cast<const uint8_t *>(buf.data())); for (size_t i(0); i < buf.size(); ++i) { dst[i] = 0xff - src[i]; diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.h b/searchlib/src/vespa/searchlib/attribute/stringbase.h index e20a40d2df3..3c85a7c318b 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.h +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.h @@ -51,6 +51,9 @@ public: static void generateOffsets(const char * bt, size_t sz, OffsetVector & offsets); virtual const char * getFromEnum(EnumHandle e) const = 0; virtual const char *get(DocId doc) const = 0; + largeint_t getInt(DocId doc) const override { return strtoll(get(doc), nullptr, 0); } + double getFloat(DocId doc) const override; + vespalib::ConstArrayRef<char> get_raw(DocId) const override; protected: StringAttribute(const vespalib::string & name); StringAttribute(const vespalib::string & name, const Config & c); @@ -79,10 +82,6 @@ private: virtual void load_enumerated_data(ReaderBase &attrReader, enumstore::EnumeratedLoader& loader); virtual void load_posting_lists_and_update_enum_store(enumstore::EnumeratedPostingsLoader& loader); - largeint_t getInt(DocId doc) const override { return strtoll(get(doc), nullptr, 0); } - double getFloat(DocId doc) const override; - const char * getString(DocId doc, char * v, size_t sz) const override { (void) v; (void) sz; return get(doc); } - long onSerializeForAscendingSort(DocId doc, void * serTo, long available, const common::BlobConverter * bc) const override; long onSerializeForDescendingSort(DocId doc, void * serTo, long available, const common::BlobConverter * bc) const override; }; diff --git a/searchlib/src/vespa/searchlib/common/bitvector.h b/searchlib/src/vespa/searchlib/common/bitvector.h index 67f3e2ad502..149e57390af 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.h +++ b/searchlib/src/vespa/searchlib/common/bitvector.h @@ -3,10 +3,8 @@ #pragma once #include "bitword.h" -#include <memory> #include <vespa/vespalib/util/alloc.h> #include <vespa/vespalib/util/atomic.h> -#include <vespa/fastos/types.h> #include <algorithm> #include <cassert> diff --git a/searchlib/src/vespa/searchlib/common/bitvectorcache.cpp b/searchlib/src/vespa/searchlib/common/bitvectorcache.cpp index cb4d112b77e..8bb24bcbbec 100644 --- a/searchlib/src/vespa/searchlib/common/bitvectorcache.cpp +++ b/searchlib/src/vespa/searchlib/common/bitvectorcache.cpp @@ -3,6 +3,7 @@ #include <vespa/vespalib/stllike/hash_map.hpp> #include <algorithm> #include <cassert> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".searchlib.common.bitvectorcache"); diff --git a/searchlib/src/vespa/searchlib/common/identifiable.h b/searchlib/src/vespa/searchlib/common/identifiable.h index 1f5aff6d2d0..4aabbadd081 100644 --- a/searchlib/src/vespa/searchlib/common/identifiable.h +++ b/searchlib/src/vespa/searchlib/common/identifiable.h @@ -151,7 +151,8 @@ #define CID_search_expression_AttributeMapLookupNode SEARCHLIB_CID(145) #define CID_search_expression_BoolResultNode SEARCHLIB_CID(146) #define CID_search_expression_BoolResultNodeVector SEARCHLIB_CID(147) - +#define CID_search_expression_IntegerAttributeResult SEARCHLIB_CID(148) +#define CID_search_expression_FloatAttributeResult SEARCHLIB_CID(149) #define CID_search_QueryNode SEARCHLIB_CID(150) #define CID_search_Query SEARCHLIB_CID(151) diff --git a/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp b/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp index e1c31c1ed8d..5399d70fbe7 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp @@ -5,6 +5,7 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/data/fileheader.h> #include <vespa/fastos/file.h> +#include <cinttypes> #include <arpa/inet.h> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/docstore/compacter.cpp b/searchlib/src/vespa/searchlib/docstore/compacter.cpp index 04ff150c741..c886e52659f 100644 --- a/searchlib/src/vespa/searchlib/docstore/compacter.cpp +++ b/searchlib/src/vespa/searchlib/docstore/compacter.cpp @@ -4,6 +4,7 @@ #include "logdatastore.h" #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/array.hpp> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docstore.compacter"); diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp index 174b6b92cd5..e76caba55a1 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp @@ -35,6 +35,7 @@ using vespalib::IllegalStateException; using vespalib::getErrorString; using vespalib::getLastErrorString; using vespalib::make_string; +using vespalib::to_string; using CpuCategory = CpuUsage::Category; @@ -665,7 +666,7 @@ lsSingleFile(const vespalib::string & fileName) vespalib::string s; FastOS_StatInfo stat; if ( FastOS_File::Stat(fileName.c_str(), &stat)) { - s += make_string("%s %20" PRIu64 " %12" PRId64, fileName.c_str(), stat._modifiedTimeNS, stat._size); + s += make_string("%s %20" PRIu64 " %12" PRId64, fileName.c_str(), vespalib::count_ns(stat._modifiedTime.time_since_epoch()), stat._size); } else { s = make_string("%s 'stat' FAILED !!", fileName.c_str()); } @@ -744,16 +745,16 @@ LogDataStore::verifyModificationTime(const NameIdSet & partList) throw runtime_error(make_string("Failed to Stat '%s'\nDirectory =\n%s", idxName.c_str(), ls(partList).c_str())); } ns_log::Logger::LogLevel logLevel = ns_log::Logger::debug; - if ((datStat._modifiedTimeNS < prevDatStat._modifiedTimeNS) && hasNonHeaderData(datName)) { - VLOG(logLevel, "Older file '%s' is newer (%" PRIu64 ") than file '%s' (%" PRIu64 ")\nDirectory =\n%s", - prevDatNam.c_str(), prevDatStat._modifiedTimeNS, - datName.c_str(), datStat._modifiedTimeNS, + if ((datStat._modifiedTime < prevDatStat._modifiedTime) && hasNonHeaderData(datName)) { + VLOG(logLevel, "Older file '%s' is newer (%s) than file '%s' (%s)\nDirectory =\n%s", + prevDatNam.c_str(), to_string(prevDatStat._modifiedTime).c_str(), + datName.c_str(), to_string(datStat._modifiedTime).c_str(), ls(partList).c_str()); } - if ((idxStat._modifiedTimeNS < prevIdxStat._modifiedTimeNS) && hasNonHeaderData(idxName)) { - VLOG(logLevel, "Older file '%s' is newer (%" PRIu64 ") than file '%s' (%" PRIu64 ")\nDirectory =\n%s", - prevIdxNam.c_str(), prevIdxStat._modifiedTimeNS, - idxName.c_str(), idxStat._modifiedTimeNS, + if ((idxStat._modifiedTime < prevIdxStat._modifiedTime) && hasNonHeaderData(idxName)) { + VLOG(logLevel, "Older file '%s' is newer (%s) than file '%s' (%s)\nDirectory =\n%s", + prevIdxNam.c_str(), to_string(prevIdxStat._modifiedTime).c_str(), + idxName.c_str(), to_string(idxStat._modifiedTime).c_str(), ls(partList).c_str()); } prevDatStat = datStat; diff --git a/searchlib/src/vespa/searchlib/engine/proto_converter.cpp b/searchlib/src/vespa/searchlib/engine/proto_converter.cpp index ae00889850b..b03dbf5ff37 100644 --- a/searchlib/src/vespa/searchlib/engine/proto_converter.cpp +++ b/searchlib/src/vespa/searchlib/engine/proto_converter.cpp @@ -6,8 +6,9 @@ #include <vespa/vespalib/data/slime/binary_format.h> #include <vespa/vespalib/data/smart_buffer.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/log/log.h> +#include <cinttypes> +#include <vespa/log/log.h> LOG_SETUP(".searchlib.engine.proto_converter"); namespace search::engine { diff --git a/searchlib/src/vespa/searchlib/expression/attributenode.cpp b/searchlib/src/vespa/searchlib/expression/attributenode.cpp index f8ae4bd698d..413cb50ca49 100644 --- a/searchlib/src/vespa/searchlib/expression/attributenode.cpp +++ b/searchlib/src/vespa/searchlib/expression/attributenode.cpp @@ -76,9 +76,12 @@ std::unique_ptr<AttributeResult> createResult(const IAttributeVector * attribute) { IAttributeVector::EnumRefs enumRefs = attribute->make_enum_read_view(); - return (enumRefs.empty()) - ? std::make_unique<AttributeResult>(attribute, 0) - : std::make_unique<EnumAttributeResult>(enumRefs, attribute, 0); + if (enumRefs.empty()) { + if (attribute->isIntegerType()) return std::make_unique<IntegerAttributeResult>(attribute, 0); + if (attribute->isFloatingPointType()) return std::make_unique<FloatAttributeResult>(attribute, 0); + return std::make_unique<AttributeResult>(attribute, 0); + } + return std::make_unique<EnumAttributeResult>(enumRefs, attribute, 0); } } @@ -221,6 +224,13 @@ AttributeNode::onPrepare(bool preserveAccurateTypes) setResultType(std::make_unique<StringResultNode>()); } } + } else if (attribute->is_raw_type()) { + if (_hasMultiValue) { + throw std::runtime_error(make_string("Does not support multivalue raw attribute vector '%s'", + attribute->getName().c_str())); + } else { + setResultType(std::make_unique<RawResultNode>()); + } } else { throw std::runtime_error(make_string("Can not deduce correct resultclass for attribute vector '%s'", attribute->getName().c_str())); diff --git a/searchlib/src/vespa/searchlib/expression/attributenode.h b/searchlib/src/vespa/searchlib/expression/attributenode.h index 03b7909e581..d668bd3f662 100644 --- a/searchlib/src/vespa/searchlib/expression/attributenode.h +++ b/searchlib/src/vespa/searchlib/expression/attributenode.h @@ -46,7 +46,7 @@ public: AttributeNode(const search::attribute::IAttributeVector & attribute); AttributeNode(const AttributeNode & attribute); AttributeNode & operator = (const AttributeNode & attribute); - ~AttributeNode(); + ~AttributeNode() override; void setDocId(DocId docId) const { _scratchResult->setDocId(docId); } const search::attribute::IAttributeVector *getAttribute() const { return _scratchResult ? _scratchResult->getAttribute() : nullptr; @@ -59,7 +59,7 @@ public: class Handler { public: - virtual ~Handler() { } + virtual ~Handler() = default; virtual void handle(const AttributeResult & r) = 0; }; private: diff --git a/searchlib/src/vespa/searchlib/expression/attributeresult.cpp b/searchlib/src/vespa/searchlib/expression/attributeresult.cpp index 9eb8b35d83c..033e782a2bb 100644 --- a/searchlib/src/vespa/searchlib/expression/attributeresult.cpp +++ b/searchlib/src/vespa/searchlib/expression/attributeresult.cpp @@ -1,9 +1,35 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attributeresult.h" +#include <charconv> namespace search::expression { IMPLEMENT_RESULTNODE(AttributeResult, ResultNode); +IMPLEMENT_RESULTNODE(IntegerAttributeResult, ResultNode); +IMPLEMENT_RESULTNODE(FloatAttributeResult, ResultNode); + +ResultNode::ConstBufferRef +IntegerAttributeResult::onGetString(size_t , BufferRef buf) const { + if (buf.size() > 1) { + char * s = buf.str(); + size_t sz = buf.size(); + long v = getAttribute()->getInt(getDocId()); + auto res = std::to_chars(s, s + sz - 1, v, 10); + if (res.ec == std::errc()) { + res.ptr[0] = 0; + } else { + s[0] = 0; + } + } + return buf; +} + +ResultNode::ConstBufferRef +FloatAttributeResult::onGetString(size_t, BufferRef buf) const { + double val = getAttribute()->getFloat(getDocId()); + int numWritten(std::min(buf.size(), (size_t)std::max(0, snprintf(buf.str(), buf.size(), "%g", val)))); + return ConstBufferRef(buf.str(), numWritten); +} } diff --git a/searchlib/src/vespa/searchlib/expression/attributeresult.h b/searchlib/src/vespa/searchlib/expression/attributeresult.h index 0501b6477cf..9907fd46ffc 100644 --- a/searchlib/src/vespa/searchlib/expression/attributeresult.h +++ b/searchlib/src/vespa/searchlib/expression/attributeresult.h @@ -13,20 +13,23 @@ public: using UP = std::unique_ptr<AttributeResult>; DECLARE_RESULTNODE(AttributeResult); AttributeResult() : _attribute(nullptr), _docId(0) { } - AttributeResult(const attribute::IAttributeVector * attribute, DocId docId) : - _attribute(attribute), - _docId(docId) + AttributeResult(const attribute::IAttributeVector * attribute, DocId docId) + : _attribute(attribute), + _docId(docId) { } void setDocId(DocId docId) { _docId = docId; } const search::attribute::IAttributeVector *getAttribute() const { return _attribute; } DocId getDocId() const { return _docId; } +protected: + ConstBufferRef get_raw() const { + auto raw = getAttribute()->get_raw(_docId); + return {raw.data(), raw.size()}; + } private: int64_t onGetInteger(size_t index) const override { (void) index; return _attribute->getInt(_docId); } double onGetFloat(size_t index) const override { (void) index; return _attribute->getFloat(_docId); } - ConstBufferRef onGetString(size_t index, BufferRef buf) const override { - (void) index; - const char * t = _attribute->getString(_docId, buf.str(), buf.size()); - return ConstBufferRef(t, strlen(t)); + ConstBufferRef onGetString(size_t, BufferRef) const override { + return get_raw(); } int64_t onGetEnum(size_t index) const override { (void) index; return (static_cast<int64_t>(_attribute->getEnum(_docId))); } void set(const search::expression::ResultNode&) override { } @@ -36,4 +39,24 @@ private: DocId _docId; }; +class IntegerAttributeResult : public AttributeResult { +public: + DECLARE_RESULTNODE(IntegerAttributeResult); + IntegerAttributeResult() : AttributeResult() {} + IntegerAttributeResult(const attribute::IAttributeVector * attribute, DocId docId) + : AttributeResult(attribute, docId) + { } + ConstBufferRef onGetString(size_t index, BufferRef buf) const override; +}; + +class FloatAttributeResult : public AttributeResult { +public: + DECLARE_RESULTNODE(FloatAttributeResult); + FloatAttributeResult() : AttributeResult() {} + FloatAttributeResult(const attribute::IAttributeVector * attribute, DocId docId) + : AttributeResult(attribute, docId) + { } + ConstBufferRef onGetString(size_t index, BufferRef buf) const override; +}; + } diff --git a/searchlib/src/vespa/searchlib/expression/catfunctionnode.h b/searchlib/src/vespa/searchlib/expression/catfunctionnode.h index 0667b408500..33df55c891a 100644 --- a/searchlib/src/vespa/searchlib/expression/catfunctionnode.h +++ b/searchlib/src/vespa/searchlib/expression/catfunctionnode.h @@ -3,8 +3,7 @@ #include "multiargfunctionnode.h" -namespace search { -namespace expression { +namespace search::expression { class CatFunctionNode : public MultiArgFunctionNode { @@ -19,5 +18,3 @@ private: }; } -} - diff --git a/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp b/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp index f48be061d15..bd13c032a03 100644 --- a/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp @@ -140,7 +140,8 @@ DocumentFieldNode::onPrepare(bool preserveAccurateTypes) } } -void DocumentFieldNode::onDocType(const DocumentType & docType) +void +DocumentFieldNode::onDocType(const DocumentType & docType) { LOG(debug, "DocumentFieldNode::onDocType(this=%p)", this); _fieldPath.clear(); @@ -173,12 +174,14 @@ private: char DefaultValue::null = 0; -void DefaultValue::set(const ResultNode&) +void +DefaultValue::set(const ResultNode&) { throw std::runtime_error("DefaultValue::set(const ResultNode&) is not possible."); } -void FieldValue2ResultNode::set(const ResultNode&) +void +FieldValue2ResultNode::set(const ResultNode&) { throw std::runtime_error("FieldValue2ResultNode::set(const ResultNode&) is not possible."); } @@ -192,7 +195,8 @@ void DocumentFieldNode::onDoc(const Document & doc) _handler->reset(); } -bool DocumentFieldNode::onExecute() const +bool +DocumentFieldNode::onExecute() const { _doc->iterateNested(_fieldPath.getFullRange(), *_handler); return true; @@ -237,12 +241,14 @@ DocumentFieldNode::Handler::onStructStart(const Content & c) } -Serializer & DocumentFieldNode::onSerialize(Serializer & os) const +Serializer & +DocumentFieldNode::onSerialize(Serializer & os) const { return os << _fieldName << _value; } -Deserializer & DocumentFieldNode::onDeserialize(Deserializer & is) +Deserializer & +DocumentFieldNode::onDeserialize(Deserializer & is) { return is >> _fieldName >> _value; } diff --git a/searchlib/src/vespa/searchlib/expression/enumresultnode.h b/searchlib/src/vespa/searchlib/expression/enumresultnode.h index 14dacd75651..6d201cb2b5d 100644 --- a/searchlib/src/vespa/searchlib/expression/enumresultnode.h +++ b/searchlib/src/vespa/searchlib/expression/enumresultnode.h @@ -3,8 +3,7 @@ #include "integerresultnode.h" -namespace search { -namespace expression { +namespace search::expression { class EnumResultNode : public IntegerResultNodeT<int64_t> { @@ -20,5 +19,3 @@ private: }; } -} - diff --git a/searchlib/src/vespa/searchlib/expression/floatresultnode.h b/searchlib/src/vespa/searchlib/expression/floatresultnode.h index c31f9a2de40..e79911fe985 100644 --- a/searchlib/src/vespa/searchlib/expression/floatresultnode.h +++ b/searchlib/src/vespa/searchlib/expression/floatresultnode.h @@ -4,8 +4,7 @@ #include "numericresultnode.h" #include <vespa/vespalib/util/sort.h> -namespace search { -namespace expression { +namespace search ::expression { class FloatResultNode final : public NumericResultNode { @@ -54,5 +53,3 @@ private: }; } -} - diff --git a/searchlib/src/vespa/searchlib/expression/integerbucketresultnode.h b/searchlib/src/vespa/searchlib/expression/integerbucketresultnode.h index 95a4555e6e4..ffd0fb11701 100644 --- a/searchlib/src/vespa/searchlib/expression/integerbucketresultnode.h +++ b/searchlib/src/vespa/searchlib/expression/integerbucketresultnode.h @@ -3,8 +3,7 @@ #include "bucketresultnode.h" -namespace search { -namespace expression { +namespace search::expression { class IntegerBucketResultNode : public BucketResultNode { @@ -48,5 +47,3 @@ public: }; } -} - diff --git a/searchlib/src/vespa/searchlib/expression/nullresultnode.h b/searchlib/src/vespa/searchlib/expression/nullresultnode.h index e873d85d0f1..b16fa2245de 100644 --- a/searchlib/src/vespa/searchlib/expression/nullresultnode.h +++ b/searchlib/src/vespa/searchlib/expression/nullresultnode.h @@ -3,8 +3,7 @@ #include "singleresultnode.h" -namespace search { -namespace expression { +namespace search::expression { class NullResultNode : public SingleResultNode { @@ -32,5 +31,3 @@ private: }; } -} - diff --git a/searchlib/src/vespa/searchlib/expression/positiveinfinityresultnode.h b/searchlib/src/vespa/searchlib/expression/positiveinfinityresultnode.h index 261b60b3613..a12bcaa0a32 100644 --- a/searchlib/src/vespa/searchlib/expression/positiveinfinityresultnode.h +++ b/searchlib/src/vespa/searchlib/expression/positiveinfinityresultnode.h @@ -3,8 +3,7 @@ #include "singleresultnode.h" -namespace search { -namespace expression { +namespace search::expression { class PositiveInfinityResultNode : public SingleResultNode { @@ -26,5 +25,3 @@ private: }; } -} - diff --git a/searchlib/src/vespa/searchlib/expression/resultnode.h b/searchlib/src/vespa/searchlib/expression/resultnode.h index 4c81259325b..6a62600b993 100644 --- a/searchlib/src/vespa/searchlib/expression/resultnode.h +++ b/searchlib/src/vespa/searchlib/expression/resultnode.h @@ -53,7 +53,6 @@ private: public: DECLARE_ABSTRACT_RESULTNODE(ResultNode); - ~ResultNode() { } using UP = std::unique_ptr<ResultNode>; using CP = vespalib::IdentifiablePtr<ResultNode>; virtual void set(const ResultNode & rhs) = 0; diff --git a/searchlib/src/vespa/searchlib/expression/stringresultnode.h b/searchlib/src/vespa/searchlib/expression/stringresultnode.h index 79d849bdd15..303d8778e99 100644 --- a/searchlib/src/vespa/searchlib/expression/stringresultnode.h +++ b/searchlib/src/vespa/searchlib/expression/stringresultnode.h @@ -3,8 +3,7 @@ #include "singleresultnode.h" -namespace search { -namespace expression { +namespace search::expression { class StringResultNode : public SingleResultNode { @@ -60,5 +59,3 @@ private: }; } -} - diff --git a/searchlib/src/vespa/searchlib/features/CMakeLists.txt b/searchlib/src/vespa/searchlib/features/CMakeLists.txt index 8acf28f4a2f..4af5c0e561e 100644 --- a/searchlib/src/vespa/searchlib/features/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/features/CMakeLists.txt @@ -7,6 +7,7 @@ vespa_add_library(searchlib_features OBJECT attributematchfeature.cpp bm25_feature.cpp closenessfeature.cpp + closest_feature.cpp constant_feature.cpp debug_attribute_wait.cpp debug_wait.cpp diff --git a/searchlib/src/vespa/searchlib/features/closenessfeature.cpp b/searchlib/src/vespa/searchlib/features/closenessfeature.cpp index e44c94dbb2d..048a507b3fd 100644 --- a/searchlib/src/vespa/searchlib/features/closenessfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/closenessfeature.cpp @@ -36,7 +36,7 @@ ConvertRawScoreToCloseness::ConvertRawScoreToCloseness(const fef::IQueryEnvironm } ConvertRawScoreToCloseness::ConvertRawScoreToCloseness(const fef::IQueryEnvironment &env, const vespalib::string &label) - : _bundle(env, label, "closeness"), + : _bundle(env, std::nullopt, label, "closeness"), _md(nullptr) { } diff --git a/searchlib/src/vespa/searchlib/features/closest_feature.cpp b/searchlib/src/vespa/searchlib/features/closest_feature.cpp new file mode 100644 index 00000000000..c0284c9fa89 --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/closest_feature.cpp @@ -0,0 +1,282 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "closest_feature.h" +#include "constant_tensor_executor.h" +#include "distance_calculator_bundle.h" +#include "valuefeature.h" +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/searchcommon/common/schema.h> +#include <vespa/searchlib/fef/indexproperties.h> +#include <vespa/searchlib/fef/parameterdescriptions.h> +#include <vespa/searchlib/fef/test/dummy_dependency_handler.h> +#include <vespa/searchlib/tensor/distance_calculator.h> +#include <vespa/searchlib/tensor/fast_value_view.h> +#include <vespa/searchlib/tensor/i_tensor_attribute.h> +#include <vespa/searchlib/tensor/serialized_tensor_ref.h> +#include <vespa/searchlib/tensor/subspace_type.h> +#include <vespa/vespalib/util/stash.h> + +#include <vespa/log/log.h> +LOG_SETUP(".features.closest_feature"); + +using search::fef::FeatureType; +using search::fef::FieldInfo; +using search::fef::ParameterDataTypeSet; +using search::tensor::FastValueView; +using search::tensor::ITensorAttribute; +using search::tensor::SubspaceType; +using search::tensor::VectorBundle; +using vespalib::eval::CellType; +using vespalib::eval::FastValueBuilderFactory; +using vespalib::eval::TypedCells; +using vespalib::eval::TypifyCellType; +using vespalib::eval::Value; +using vespalib::eval::ValueType; +using vespalib::string_id; +using vespalib::typify_invoke; + +using namespace search::fef::indexproperties; + +namespace { + +struct SetIdentity { + template <typename T> + static void invoke(void *space, size_t size) { + assert(size == sizeof(T)); + *(T *) space = 1.0; + } +}; + +void setup_identity_cells(const ValueType& type, std::vector<char>& space, TypedCells& cells) +{ + if (type.is_double()) { + return; + } + space.resize(vespalib::eval::CellTypeUtils::mem_size(type.cell_type(), 1)); + cells = TypedCells(space.data(), type.cell_type(), 1); + typify_invoke<1,TypifyCellType,SetIdentity>(type.cell_type(), space.data(), space.size()); +} + +} + +namespace search::features { + +class ClosestExecutor : public fef::FeatureExecutor { +protected: + DistanceCalculatorBundle _bundle; + Value& _empty_output; + TypedCells _identity; + const ITensorAttribute& _attr; + std::unique_ptr<Value> _output; +public: + ClosestExecutor(DistanceCalculatorBundle&& bundle, Value& empty_output, TypedCells identity, const ITensorAttribute& attr); + ~ClosestExecutor() override; + static fef::FeatureExecutor& make(DistanceCalculatorBundle&& bundle, Value& empty_output, TypedCells identity, const ITensorAttribute& attr, vespalib::Stash& stash); +}; + +/** + * Implements the executor for the closest feature for SerializedFastValueAttribute. + */ +class ClosestSerializedExecutor : public ClosestExecutor { +public: + ClosestSerializedExecutor(DistanceCalculatorBundle&& bundle, Value& empty_output, TypedCells identity, const ITensorAttribute& attr); + ~ClosestSerializedExecutor() override; + void execute(uint32_t docId) override; +}; + +/** + * Implements the executor for the closest feature for DirectTensorAttribute. + */ +class ClosestDirectExecutor : public ClosestExecutor { + SubspaceType _subspace_type; + std::vector<string_id> _labels; + std::vector<string_id*> _label_ptrs; +public: + ClosestDirectExecutor(DistanceCalculatorBundle&& bundle, Value& empty_output, TypedCells identity, const ITensorAttribute& attr); + ~ClosestDirectExecutor() override; + void execute(uint32_t docId) override; +}; + +ClosestExecutor::ClosestExecutor(DistanceCalculatorBundle&& bundle, Value& empty_output, TypedCells identity, const ITensorAttribute& attr) + : _bundle(std::move(bundle)), + _empty_output(empty_output), + _identity(identity), + _attr(attr), + _output() +{ +} + +ClosestExecutor::~ClosestExecutor() = default; + +fef::FeatureExecutor& +ClosestExecutor::make(DistanceCalculatorBundle&& bundle, Value& empty_output, TypedCells identity, const ITensorAttribute& attr, vespalib::Stash& stash) +{ + if (attr.supports_get_serialized_tensor_ref()) { + return stash.create<ClosestSerializedExecutor>(std::move(bundle), empty_output, identity, attr); + } else if (attr.supports_get_tensor_ref()) { + return stash.create<ClosestDirectExecutor>(std::move(bundle), empty_output, identity, attr); + } else { + return ConstantTensorExecutor::createEmpty(empty_output.type(), stash); + } +} + +ClosestSerializedExecutor::ClosestSerializedExecutor(DistanceCalculatorBundle&& bundle, Value& empty_output, TypedCells identity, const ITensorAttribute& attr) + : ClosestExecutor(std::move(bundle), empty_output, identity, attr) +{ +} + +ClosestSerializedExecutor::~ClosestSerializedExecutor() = default; + +void +ClosestSerializedExecutor::execute(uint32_t docId) +{ + double best_distance = 0.0; + std::optional<uint32_t> closest_subspace; + auto ref = _attr.get_serialized_tensor_ref(docId); + for (const auto& elem : _bundle.elements()) { + elem.calc->calc_closest_subspace(ref.get_vectors(), closest_subspace, best_distance); + } + if (closest_subspace.has_value()) { + auto labels = ref.get_labels(closest_subspace.value()); + _output = std::make_unique<FastValueView>(_empty_output.type(), labels, _identity, labels.size(), 1); + outputs().set_object(0, *_output); + } else { + outputs().set_object(0, _empty_output); + } +} + +ClosestDirectExecutor::ClosestDirectExecutor(DistanceCalculatorBundle&& bundle, Value& empty_output, TypedCells identity, const ITensorAttribute& attr) + : ClosestExecutor(std::move(bundle), empty_output, identity, attr), + _subspace_type(attr.getTensorType()), + _labels(1), + _label_ptrs(_labels.size()) +{ + for (size_t i = 0; i < _labels.size(); ++i) { + _label_ptrs[i] = &_labels[i]; + } +} + +ClosestDirectExecutor::~ClosestDirectExecutor() = default; + +void +ClosestDirectExecutor::execute(uint32_t docId) +{ + double best_distance = 0.0; + std::optional<uint32_t> closest_subspace; + auto& tensor = _attr.get_tensor_ref(docId); + VectorBundle vectors(tensor.cells().data, tensor.index().size(), _subspace_type); + for (const auto& elem : _bundle.elements()) { + elem.calc->calc_closest_subspace(vectors, closest_subspace, best_distance); + } + if (closest_subspace.has_value()) { + size_t subspace_id = 0; + auto view = tensor.index().create_view({}); + view->lookup({}); + while (view->next_result(_label_ptrs, subspace_id)) { + if (subspace_id == closest_subspace.value()) { + _output = std::make_unique<FastValueView>(_empty_output.type(), _labels, _identity, _labels.size(), 1); + outputs().set_object(0, *_output); + return; + } + } + } + outputs().set_object(0, _empty_output); +} + +ClosestBlueprint::ClosestBlueprint() + : Blueprint("closest"), + _field_name(), + _field_tensor_type(ValueType::error_type()), + _output_tensor_type(ValueType::error_type()), + _field_id(search::index::Schema::UNKNOWN_FIELD_ID), + _item_label(), + _empty_output(), + _identity_space(), + _identity_cells() +{ +} + +ClosestBlueprint::~ClosestBlueprint() = default; + +void +ClosestBlueprint::visitDumpFeatures(const fef::IIndexEnvironment&, fef::IDumpFeatureVisitor&) const +{ +} + +std::unique_ptr<fef::Blueprint> +ClosestBlueprint::createInstance() const +{ + return std::make_unique<ClosestBlueprint>(); +} + +fef::ParameterDescriptions +ClosestBlueprint::getDescriptions() const +{ + auto data_type_set = ParameterDataTypeSet::tensor_type_set(); + return fef::ParameterDescriptions(). + desc().attribute(data_type_set, fef::ParameterCollection::SINGLE). + desc().attribute(data_type_set, fef::ParameterCollection::SINGLE).string(); +} + +bool +ClosestBlueprint::setup(const fef::IIndexEnvironment & env, const fef::ParameterList & params) +{ + if (params.size() < 1 || params.size() > 2) { + LOG(error, "%s: Wrong number of parameters, was %d, must be 1 or 2", getName().c_str(), (int) params.size()); + return false; + } + _field_name = params[0].getValue(); + if (params.size() == 2) { + _item_label = params[1].getValue(); + } + auto fi = env.getFieldByName(_field_name); + assert(fi != nullptr); + vespalib::string attr_type_spec = type::Attribute::lookup(env.getProperties(), _field_name); + if (attr_type_spec.empty()) { + LOG(error, "%s: Field %s lacks a type in index properties", getName().c_str(), _field_name.c_str()); + return false; + } + _field_tensor_type = ValueType::from_spec(attr_type_spec); + if (_field_tensor_type.is_error() || _field_tensor_type.is_double() || _field_tensor_type.count_mapped_dimensions() != 1 || _field_tensor_type.count_indexed_dimensions() != 1) { + LOG(error, "%s: Field %s has invalid type: '%s'", getName().c_str(), _field_name.c_str(), attr_type_spec.c_str()); + return false; + } + _output_tensor_type = ValueType::make_type(_field_tensor_type.cell_type(), _field_tensor_type.mapped_dimensions()); + assert(!_output_tensor_type.is_double()); + FeatureType output_type = FeatureType::object(_output_tensor_type); + describeOutput("out", "The closest tensor subspace.", output_type); + _field_id = fi->id(); + _empty_output = vespalib::eval::value_from_spec(_output_tensor_type.to_spec(), FastValueBuilderFactory::get()); + setup_identity_cells(_output_tensor_type, _identity_space, _identity_cells); + return true; +} + +void +ClosestBlueprint::prepareSharedState(const fef::IQueryEnvironment& env, fef::IObjectStore& store) const +{ + if (_item_label.has_value()) { + DistanceCalculatorBundle::prepare_shared_state(env, store, _item_label.value(), "closest"); + } else { + DistanceCalculatorBundle::prepare_shared_state(env, store, _field_id, "closest"); + } +} + +fef::FeatureExecutor& +ClosestBlueprint::createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const +{ + auto bundle = _item_label.has_value() ? DistanceCalculatorBundle(env, _field_id, _item_label.value(), "closest") : DistanceCalculatorBundle(env, _field_id, "closest"); + if (bundle.elements().empty()) { + return ConstantTensorExecutor::createEmpty(_output_tensor_type, stash); + } else { + for (const auto& elem : bundle.elements()) { + if (!elem.calc) { + return ConstantTensorExecutor::createEmpty(_output_tensor_type, stash); + } + } + auto& attr = bundle.elements().front().calc->attribute_tensor(); + return ClosestExecutor::make(std::move(bundle), *_empty_output, _identity_cells, attr, stash); + } +} + +} diff --git a/searchlib/src/vespa/searchlib/features/closest_feature.h b/searchlib/src/vespa/searchlib/features/closest_feature.h new file mode 100644 index 00000000000..840f896abe2 --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/closest_feature.h @@ -0,0 +1,33 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/fef/blueprint.h> +#include <optional> + +namespace search::features { + +/** + * Implements the blueprint for the closest executor. + */ +class ClosestBlueprint : public fef::Blueprint { + vespalib::string _field_name; + vespalib::eval::ValueType _field_tensor_type; + vespalib::eval::ValueType _output_tensor_type; + uint32_t _field_id; + std::optional<vespalib::string> _item_label; + std::unique_ptr<vespalib::eval::Value> _empty_output; + std::vector<char> _identity_space; + vespalib::eval::TypedCells _identity_cells; +public: + ClosestBlueprint(); + ~ClosestBlueprint() override; + void visitDumpFeatures(const fef::IIndexEnvironment & env, fef::IDumpFeatureVisitor & visitor) const override; + std::unique_ptr<fef::Blueprint> createInstance() const override; + fef::ParameterDescriptions getDescriptions() const override; + bool setup(const fef::IIndexEnvironment & env, const fef::ParameterList & params) override; + void prepareSharedState(const fef::IQueryEnvironment& env, fef::IObjectStore& store) const override; + fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.cpp b/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.cpp index 4b2d67c933d..fad4c649165 100644 --- a/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.cpp +++ b/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.cpp @@ -115,6 +115,7 @@ DistanceCalculatorBundle::DistanceCalculatorBundle(const fef::IQueryEnvironment& } DistanceCalculatorBundle::DistanceCalculatorBundle(const fef::IQueryEnvironment& env, + std::optional<uint32_t> field_id, const vespalib::string& label, const vespalib::string& feature_name) : _elems() @@ -124,6 +125,9 @@ DistanceCalculatorBundle::DistanceCalculatorBundle(const fef::IQueryEnvironment& // expect numFields() == 1 for (uint32_t i = 0; i < term->numFields(); ++i) { const auto& term_field = term->field(i); + if (field_id.has_value() && field_id.value() != term_field.getFieldId()) { + continue; + } TermFieldHandle handle = term_field.getHandle(); if (handle != IllegalHandle) { std::unique_ptr<DistanceCalculator> calc; diff --git a/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.h b/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.h index 35295c771a6..e3be52aecc5 100644 --- a/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.h +++ b/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.h @@ -5,6 +5,7 @@ #include <vespa/searchlib/fef/handle.h> #include <vespa/vespalib/stllike/string.h> #include <memory> +#include <optional> #include <vector> namespace search::tensor { class DistanceCalculator; } @@ -40,6 +41,7 @@ public: const vespalib::string& feature_name); DistanceCalculatorBundle(const fef::IQueryEnvironment& env, + std::optional<uint32_t> field_id, const vespalib::string& label, const vespalib::string& feature_name); diff --git a/searchlib/src/vespa/searchlib/features/distancefeature.cpp b/searchlib/src/vespa/searchlib/features/distancefeature.cpp index 40f994c18e9..f601c91a0b2 100644 --- a/searchlib/src/vespa/searchlib/features/distancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/distancefeature.cpp @@ -44,7 +44,7 @@ ConvertRawscoreToDistance::ConvertRawscoreToDistance(const fef::IQueryEnvironmen } ConvertRawscoreToDistance::ConvertRawscoreToDistance(const fef::IQueryEnvironment &env, const vespalib::string &label) - : _bundle(env, label, "distance"), + : _bundle(env, std::nullopt, label, "distance"), _md(nullptr) { } diff --git a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp index 91ccd390692..ad92f8ef6e0 100644 --- a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stash.h> #include <chrono> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".features.randomnormalfeature"); diff --git a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp index f7fcffca8cb..f1a42da1266 100644 --- a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp @@ -4,6 +4,7 @@ #include "utils.h" #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stash.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".features.randomnormalstablefeature"); diff --git a/searchlib/src/vespa/searchlib/features/randomfeature.cpp b/searchlib/src/vespa/searchlib/features/randomfeature.cpp index a8f2dd275a7..30a313d54d2 100644 --- a/searchlib/src/vespa/searchlib/features/randomfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/randomfeature.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stash.h> #include <chrono> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".features.randomfeature"); diff --git a/searchlib/src/vespa/searchlib/features/setup.cpp b/searchlib/src/vespa/searchlib/features/setup.cpp index 2bc8a349d1b..5e152d4b455 100644 --- a/searchlib/src/vespa/searchlib/features/setup.cpp +++ b/searchlib/src/vespa/searchlib/features/setup.cpp @@ -6,6 +6,7 @@ #include "attributematchfeature.h" #include "bm25_feature.h" #include "closenessfeature.h" +#include "closest_feature.h" #include "constant_feature.h" #include "debug_attribute_wait.h" #include "debug_wait.h" @@ -75,6 +76,7 @@ void setup_search_features(fef::IBlueprintRegistry & registry) registry.addPrototype(std::make_shared<AttributeMatchBlueprint>()); registry.addPrototype(std::make_shared<Bm25Blueprint>()); registry.addPrototype(std::make_shared<ClosenessBlueprint>()); + registry.addPrototype(std::make_shared<ClosestBlueprint>()); registry.addPrototype(std::make_shared<DebugAttributeWaitBlueprint>()); registry.addPrototype(std::make_shared<DebugWaitBlueprint>()); registry.addPrototype(std::make_shared<DistanceBlueprint>()); diff --git a/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h b/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h index e47ce0df7a5..46a932696ca 100644 --- a/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h +++ b/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h @@ -71,6 +71,7 @@ private: asMask(DataType::REFERENCE) | asMask(DataType::COMBINED)); } + static uint32_t tensor_type_mask() { return asMask(DataType::TENSOR); } ParameterDataTypeSet(uint32_t typeMask) : _typeMask(typeMask) { @@ -87,8 +88,9 @@ public: return ParameterDataTypeSet(asMask(DataType::INT32) | asMask(DataType::INT64)); } static ParameterDataTypeSet normalOrTensorTypeSet() { - return ParameterDataTypeSet(normalTypesMask() | asMask(DataType::TENSOR)); + return ParameterDataTypeSet(normalTypesMask() | tensor_type_mask()); } + static ParameterDataTypeSet tensor_type_set() { return ParameterDataTypeSet(tensor_type_mask()); } bool allowedType(DataType dataType) const { return ((asMask(dataType) & _typeMask) != 0); } diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h b/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h index 0bf33f1d0e5..0de9be332fa 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h +++ b/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h @@ -3,7 +3,8 @@ #include <memory> #include <cstdint> -#include <vespa/fastos/types.h> + +#define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden"))) /** * Interface for posting lists used by PredicateSearch. @@ -25,7 +26,7 @@ protected: public: using UP = std::unique_ptr<PredicatePostingList>; - virtual ~PredicatePostingList() {} + virtual ~PredicatePostingList() = default; /* * Moves to next document after the one supplied. diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h b/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h index 42de8646091..6a6b91e20f2 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h +++ b/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h @@ -5,6 +5,7 @@ #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/issue.h> #include <climits> +#include <cinttypes> namespace search::predicate { diff --git a/searchlib/src/vespa/searchlib/queryeval/hitcollector.h b/searchlib/src/vespa/searchlib/queryeval/hitcollector.h index fe686f3e0bf..e244c856c4a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/hitcollector.h +++ b/searchlib/src/vespa/searchlib/queryeval/hitcollector.h @@ -9,7 +9,6 @@ #include <vespa/vespalib/util/sort.h> #include <algorithm> #include <vector> -#include <vespa/fastos/types.h> namespace search::queryeval { diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index a00a50f32c8..c8c5d4d4257 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -3,7 +3,6 @@ vespa_add_library(searchlib_tensor OBJECT SOURCES angular_distance.cpp bitvector_visited_tracker.cpp - blob_sequence_reader.cpp default_nearest_neighbor_index_factory.cpp dense_tensor_attribute.cpp dense_tensor_store.cpp @@ -13,6 +12,7 @@ vespa_add_library(searchlib_tensor OBJECT distance_function_factory.cpp empty_subspace.cpp euclidean_distance.cpp + fast_value_view.cpp geo_degrees_distance.cpp hamming_distance.cpp hash_set_visited_tracker.cpp @@ -30,6 +30,7 @@ vespa_add_library(searchlib_tensor OBJECT nearest_neighbor_index.cpp nearest_neighbor_index_saver.cpp serialized_fast_value_attribute.cpp + serialized_tensor_ref.cpp small_subspaces_buffer_type.cpp subspace_type.cpp tensor_attribute.cpp diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.h index d331cdca440..73bd929aee6 100644 --- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.h @@ -14,6 +14,7 @@ class DirectTensorAttribute final : public TensorAttribute { DirectTensorStore _direct_store; + void set_tensor(DocId docId, std::unique_ptr<vespalib::eval::Value> tensor); public: DirectTensorAttribute(vespalib::stringref baseFileName, const Config &cfg, const NearestNeighborIndexFactory& index_factory = DefaultNearestNeighborIndexFactory()); ~DirectTensorAttribute() override; @@ -21,7 +22,6 @@ public: void update_tensor(DocId docId, const document::TensorUpdate &update, bool create_empty_if_non_existing) override; - void set_tensor(DocId docId, std::unique_ptr<vespalib::eval::Value> tensor); const vespalib::eval::Value &get_tensor_ref(DocId docId) const override; bool supports_get_tensor_ref() const override { return true; } diff --git a/searchlib/src/vespa/searchlib/tensor/distance_calculator.h b/searchlib/src/vespa/searchlib/tensor/distance_calculator.h index 320f071cbbb..6b4cf142264 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_calculator.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_calculator.h @@ -4,6 +4,7 @@ #include "distance_function.h" #include "i_tensor_attribute.h" #include "vector_bundle.h" +#include <optional> namespace vespalib::eval { struct Value; } @@ -64,6 +65,23 @@ public: return result; } + void calc_closest_subspace(VectorBundle vectors, std::optional<uint32_t>& closest_subspace, double& best_distance) { + for (uint32_t i = 0; i < vectors.subspaces(); ++i) { + double distance = _dist_fun->calc(_query_tensor_cells, vectors.cells(i)); + if (!closest_subspace.has_value() || distance < best_distance) { + best_distance = distance; + closest_subspace = i; + } + } + } + + std::optional<uint32_t> calc_closest_subspace(VectorBundle vectors) { + double best_distance = 0.0; + std::optional<uint32_t> closest_subspace; + calc_closest_subspace(vectors, closest_subspace, best_distance); + return closest_subspace; + } + /** * Create a calculator for the given attribute tensor and query tensor, if possible. * diff --git a/searchlib/src/vespa/searchlib/tensor/fast_value_view.cpp b/searchlib/src/vespa/searchlib/tensor/fast_value_view.cpp new file mode 100644 index 00000000000..29cc47cb543 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/fast_value_view.cpp @@ -0,0 +1,39 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "fast_value_view.h" +#include <vespa/vespalib/stllike/hash_map.hpp> + +using vespalib::ConstArrayRef; +using vespalib::MemoryUsage; +using vespalib::string_id; +using vespalib::eval::FastAddrMap; +using vespalib::eval::TypedCells; +using vespalib::eval::Value; +using vespalib::eval::ValueType; +using vespalib::eval::self_memory_usage; + +namespace search::tensor { + +FastValueView::FastValueView(const ValueType& type, ConstArrayRef<string_id> labels, TypedCells cells, size_t num_mapped_dimensions, size_t num_subspaces) + : Value(), + _type(type), + _labels(labels.begin(), labels.end()), + _index(num_mapped_dimensions, _labels, num_subspaces), + _cells(cells) +{ + for (size_t i = 0; i < num_subspaces; ++i) { + ConstArrayRef<string_id> addr(_labels.data() + (i * num_mapped_dimensions), num_mapped_dimensions); + _index.map.add_mapping(FastAddrMap::hash_labels(addr)); + } + assert(_index.map.size() == num_subspaces); +} + +MemoryUsage +FastValueView::get_memory_usage() const +{ + MemoryUsage usage = self_memory_usage<FastValueView>(); + usage.merge(_index.map.estimate_extra_memory_usage()); + return usage; +} + +} diff --git a/searchlib/src/vespa/searchlib/tensor/fast_value_view.h b/searchlib/src/vespa/searchlib/tensor/fast_value_view.h new file mode 100644 index 00000000000..c3f13ac5856 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/fast_value_view.h @@ -0,0 +1,24 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/fast_value_index.h> + +namespace search::tensor { + +/* + * Tensor view that is not self-contained. It references external cell values. + */ +struct FastValueView final : vespalib::eval::Value { + const vespalib::eval::ValueType& _type; + vespalib::StringIdVector _labels; + vespalib::eval::FastValueIndex _index; + vespalib::eval::TypedCells _cells; + FastValueView(const vespalib::eval::ValueType& type, vespalib::ConstArrayRef<vespalib::string_id> labels, vespalib::eval::TypedCells cells, size_t num_mapped_dimensions, size_t num_subspaces); + const vespalib::eval::ValueType& type() const override { return _type; } + const vespalib::eval::Value::Index& index() const override { return _index; } + vespalib::eval::TypedCells cells() const override { return _cells; } + vespalib::MemoryUsage get_memory_usage() const override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h index 9b5f80b2ece..ec6774c9517 100644 --- a/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h @@ -13,6 +13,7 @@ namespace vespalib::slime { struct Inserter; } namespace search::tensor { class NearestNeighborIndex; +class SerializedTensorRef; /** * Interface for tensor attribute used by feature executors to get information. @@ -24,8 +25,10 @@ public: virtual std::unique_ptr<vespalib::eval::Value> getEmptyTensor() const = 0; virtual vespalib::eval::TypedCells extract_cells_ref(uint32_t docid) const = 0; virtual const vespalib::eval::Value& get_tensor_ref(uint32_t docid) const = 0; + virtual SerializedTensorRef get_serialized_tensor_ref(uint32_t docid) const = 0; virtual bool supports_extract_cells_ref() const = 0; virtual bool supports_get_tensor_ref() const = 0; + virtual bool supports_get_serialized_tensor_ref() const = 0; virtual const vespalib::eval::ValueType & getTensorType() const = 0; diff --git a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp index f9459823ce4..9a7b81ae1fa 100644 --- a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp +++ b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "imported_tensor_attribute_vector_read_guard.h" +#include "serialized_tensor_ref.h" #include "vector_bundle.h" #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/eval/eval/value.h> @@ -79,6 +80,18 @@ ImportedTensorAttributeVectorReadGuard::getTensorType() const return _target_tensor_attribute.getTensorType(); } +SerializedTensorRef +ImportedTensorAttributeVectorReadGuard::get_serialized_tensor_ref(uint32_t docid) const +{ + return _target_tensor_attribute.get_serialized_tensor_ref(getTargetLid(docid)); +} + +bool +ImportedTensorAttributeVectorReadGuard::supports_get_serialized_tensor_ref() const +{ + return _target_tensor_attribute.supports_get_serialized_tensor_ref(); +} + void ImportedTensorAttributeVectorReadGuard::get_state(const vespalib::slime::Inserter& inserter) const { diff --git a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h index f277d39e97d..4e1cc9efd96 100644 --- a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h +++ b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h @@ -35,9 +35,11 @@ public: std::unique_ptr<vespalib::eval::Value> getEmptyTensor() const override; vespalib::eval::TypedCells extract_cells_ref(uint32_t docid) const override; const vespalib::eval::Value& get_tensor_ref(uint32_t docid) const override; + SerializedTensorRef get_serialized_tensor_ref(uint32_t docid) const override; bool supports_extract_cells_ref() const override { return _target_tensor_attribute.supports_extract_cells_ref(); } bool supports_get_tensor_ref() const override { return _target_tensor_attribute.supports_get_tensor_ref(); } DistanceMetric distance_metric() const override { return _target_tensor_attribute.distance_metric(); } + bool supports_get_serialized_tensor_ref() const override; uint32_t get_num_docs() const override { return getNumDocs(); } vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const override; diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp index 6612db1d27e..51ebc22c269 100644 --- a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "serialized_fast_value_attribute.h" +#include "serialized_tensor_ref.h" #include <vespa/eval/eval/value.h> #include <vespa/searchcommon/attribute/config.h> @@ -26,6 +27,19 @@ SerializedFastValueAttribute::~SerializedFastValueAttribute() _tensorStore.reclaim_all_memory(); } +SerializedTensorRef +SerializedFastValueAttribute::get_serialized_tensor_ref(uint32_t docid) const +{ + EntryRef ref = acquire_entry_ref(docid); + return _tensorBufferStore.get_serialized_tensor_ref(ref); +} + +bool +SerializedFastValueAttribute::supports_get_serialized_tensor_ref() const +{ + return true; +} + vespalib::eval::TypedCells SerializedFastValueAttribute::get_vector(uint32_t docid, uint32_t subspace) const { diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.h b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.h index 4cfcc3d19a2..9066766fbc4 100644 --- a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.h @@ -23,6 +23,9 @@ public: SerializedFastValueAttribute(vespalib::stringref baseFileName, const Config &cfg, const NearestNeighborIndexFactory& index_factory = DefaultNearestNeighborIndexFactory()); ~SerializedFastValueAttribute() override; + SerializedTensorRef get_serialized_tensor_ref(uint32_t docid) const override; + bool supports_get_serialized_tensor_ref() const override; + // Implements DocVectorAccess vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const override; VectorBundle get_vectors(uint32_t docid) const override; diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_tensor_ref.cpp b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_ref.cpp new file mode 100644 index 00000000000..1f8ca9ed2fd --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_ref.cpp @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "serialized_tensor_ref.h" + +namespace search::tensor { + +SerializedTensorRef::SerializedTensorRef() + : _vectors(), + _num_mapped_dimensions(0), + _labels() +{ +} + +SerializedTensorRef::SerializedTensorRef(VectorBundle vectors, uint32_t num_mapped_dimensions, vespalib::ConstArrayRef<vespalib::string_id> labels) + : _vectors(vectors), + _num_mapped_dimensions(num_mapped_dimensions), + _labels(labels) +{ +} + +SerializedTensorRef::~SerializedTensorRef() = default; + +vespalib::ConstArrayRef<vespalib::string_id> +SerializedTensorRef::get_labels(uint32_t subspace) const +{ + assert(subspace < _vectors.subspaces()); + return {_labels.data() + subspace * _num_mapped_dimensions, _num_mapped_dimensions}; +} + +} diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_tensor_ref.h b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_ref.h new file mode 100644 index 00000000000..01ddaadb2ff --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_ref.h @@ -0,0 +1,26 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "vector_bundle.h" +#include <vespa/vespalib/util/string_id.h> + +namespace search::tensor { + +/* + * This class contains a reference to a tensor stored in a TensorBufferStore. + */ +class SerializedTensorRef +{ + VectorBundle _vectors; + uint32_t _num_mapped_dimensions; + vespalib::ConstArrayRef<vespalib::string_id> _labels; // all subspaces +public: + SerializedTensorRef(); + SerializedTensorRef(VectorBundle vectors, uint32_t num_mapped_dimensions, vespalib::ConstArrayRef<vespalib::string_id> labels); + ~SerializedTensorRef(); + const VectorBundle& get_vectors() const noexcept { return _vectors; } + vespalib::ConstArrayRef<vespalib::string_id> get_labels(uint32_t subspace) const; +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp index 9ee8d9fdf46..13dad7fc1f2 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp @@ -4,6 +4,7 @@ #include "nearest_neighbor_index.h" #include "nearest_neighbor_index_factory.h" #include "nearest_neighbor_index_saver.h" +#include "serialized_tensor_ref.h" #include "tensor_attribute_constants.h" #include "tensor_attribute_loader.h" #include "tensor_attribute_saver.h" @@ -261,6 +262,18 @@ TensorAttribute::get_tensor_ref(uint32_t /*docid*/) const notImplemented(); } +SerializedTensorRef +TensorAttribute::get_serialized_tensor_ref(uint32_t) const +{ + notImplemented(); +} + +bool +TensorAttribute::supports_get_serialized_tensor_ref() const +{ + return false; +} + const vespalib::eval::ValueType & TensorAttribute::getTensorType() const { diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h index a4c30a574e5..20c8ae60107 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h @@ -63,8 +63,10 @@ public: std::unique_ptr<vespalib::eval::Value> getEmptyTensor() const override; vespalib::eval::TypedCells extract_cells_ref(uint32_t docid) const override; const vespalib::eval::Value& get_tensor_ref(uint32_t docid) const override; + SerializedTensorRef get_serialized_tensor_ref(uint32_t docid) const override; bool supports_extract_cells_ref() const override { return false; } bool supports_get_tensor_ref() const override { return false; } + bool supports_get_serialized_tensor_ref() const override; const vespalib::eval::ValueType & getTensorType() const override; const NearestNeighborIndex* nearest_neighbor_index() const override; void get_state(const vespalib::slime::Inserter& inserter) const override; diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute_loader.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute_loader.cpp index 5379246be90..aada583627b 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute_loader.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute_loader.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "tensor_attribute_loader.h" -#include "blob_sequence_reader.h" #include "dense_tensor_store.h" #include "nearest_neighbor_index.h" #include "nearest_neighbor_index_loader.h" @@ -10,6 +9,7 @@ #include <vespa/fastlib/io/bufferedfile.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/attribute/attribute_header.h> +#include <vespa/searchlib/attribute/blob_sequence_reader.h> #include <vespa/searchlib/attribute/load_utils.h> #include <vespa/searchlib/attribute/readerbase.h> #include <vespa/vespalib/util/arrayqueue.hpp> @@ -22,6 +22,7 @@ LOG_SETUP(".searchlib.tensor.tensor_attribute_loader"); using search::attribute::AttributeHeader; +using search::attribute::BlobSequenceReader; using search::attribute::LoadUtils; using vespalib::CpuUsage; using vespalib::datastore::EntryRef; diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute_loader.h b/searchlib/src/vespa/searchlib/tensor/tensor_attribute_loader.h index 9417737cec5..996819b8b4e 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute_loader.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute_loader.h @@ -5,11 +5,12 @@ #include <vespa/vespalib/datastore/atomic_entry_ref.h> #include <vespa/vespalib/util/rcuvector.h> +namespace search::attribute { class BlobSequenceReader; } + namespace vespalib { class Executor; } namespace search::tensor { -class BlobSequenceReader; class DenseTensorStore; class NearestNeighborIndex; class TensorAttribute; @@ -29,8 +30,8 @@ class TensorAttributeLoader { TensorStore& _store; NearestNeighborIndex* _index; - void load_dense_tensor_store(BlobSequenceReader& reader, uint32_t docid_limit, DenseTensorStore& dense_store); - void load_tensor_store(BlobSequenceReader& reader, uint32_t docid_limit); + void load_dense_tensor_store(search::attribute::BlobSequenceReader& reader, uint32_t docid_limit, DenseTensorStore& dense_store); + void load_tensor_store(search::attribute::BlobSequenceReader& reader, uint32_t docid_limit); void build_index(vespalib::Executor* executor, uint32_t docid_limit); bool load_index(); diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.cpp index fcdb9311ec6..135c62b3cfa 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.cpp @@ -1,8 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "tensor_buffer_operations.h" -#include <vespa/eval/eval/fast_value.hpp> -#include <vespa/eval/eval/value.h> +#include "fast_value_view.h" #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/eval/value_type.h> #include <vespa/eval/streamed/streamed_value_view.h> @@ -35,36 +34,6 @@ adjust_min_alignment(size_t min_alignment) return std::max(std::max(sizeof(uint32_t), sizeof(string_id)), min_alignment); } -struct FastValueView final : Value { - const ValueType& _type; - StringIdVector _labels; - FastValueIndex _index; - TypedCells _cells; - FastValueView(const ValueType& type, ConstArrayRef<string_id> labels, TypedCells cells, size_t num_mapped_dimensions, size_t num_subspaces); - const ValueType& type() const override { return _type; } - const Value::Index& index() const override { return _index; } - TypedCells cells() const override { return _cells; } - MemoryUsage get_memory_usage() const override { - MemoryUsage usage = self_memory_usage<FastValueView>(); - usage.merge(_index.map.estimate_extra_memory_usage()); - return usage; - } -}; - -FastValueView::FastValueView(const ValueType& type, ConstArrayRef<string_id> labels, TypedCells cells, size_t num_mapped_dimensions, size_t num_subspaces) - : Value(), - _type(type), - _labels(labels.begin(), labels.end()), - _index(num_mapped_dimensions, _labels, num_subspaces), - _cells(cells) -{ - for (size_t i = 0; i < num_subspaces; ++i) { - ConstArrayRef<string_id> addr(_labels.data() + (i * num_mapped_dimensions), num_mapped_dimensions); - _index.map.add_mapping(FastAddrMap::hash_labels(addr)); - } - assert(_index.map.size() == num_subspaces); -} - } TensorBufferOperations::TensorBufferOperations(const vespalib::eval::ValueType& tensor_type) diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h index 3928b41c2d1..72940cbd6a0 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h @@ -3,6 +3,7 @@ #pragma once #include "empty_subspace.h" +#include "serialized_tensor_ref.h" #include "subspace_type.h" #include "vector_bundle.h" #include <vespa/vespalib/datastore/aligner.h> @@ -110,6 +111,13 @@ public: auto aligner = select_aligner(cells_mem_size); return VectorBundle(buf.data() + get_cells_offset(num_subspaces, aligner), num_subspaces, _subspace_type); } + SerializedTensorRef get_serialized_tensor_ref(vespalib::ConstArrayRef<char> buf) const { + auto num_subspaces = get_num_subspaces(buf); + auto cells_mem_size = get_cells_mem_size(num_subspaces); + auto aligner = select_aligner(cells_mem_size); + vespalib::ConstArrayRef<vespalib::string_id> labels(reinterpret_cast<const vespalib::string_id*>(buf.data() + get_labels_offset()), num_subspaces * _num_mapped_dimensions); + return SerializedTensorRef(VectorBundle(buf.data() + get_cells_offset(num_subspaces, aligner), num_subspaces, _subspace_type), _num_mapped_dimensions, labels); + } }; } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.h b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.h index f602836bd32..2e86ff5fb67 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.h @@ -44,6 +44,13 @@ public: auto buf = _array_store.get(ref); return _ops.get_vectors(buf); } + SerializedTensorRef get_serialized_tensor_ref(EntryRef ref) const { + if (!ref.valid()) { + return SerializedTensorRef(); + } + auto buf = _array_store.get(ref); + return _ops.get_serialized_tensor_ref(buf); + } // Used by unit test static constexpr uint32_t get_offset_bits() noexcept { return RefType::offset_bits; } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.cpp index ce8cc11026c..3bd9f72c73b 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.cpp @@ -9,13 +9,13 @@ namespace search::tensor { TensorBufferTypeMapper::TensorBufferTypeMapper() - : _array_sizes(), + : vespalib::datastore::ArrayStoreTypeMapper(), _ops(nullptr) { } TensorBufferTypeMapper::TensorBufferTypeMapper(uint32_t max_small_subspaces_type_id, double grow_factor, TensorBufferOperations* ops) - : _array_sizes(), + : vespalib::datastore::ArrayStoreTypeMapper(), _ops(ops) { _array_sizes.reserve(max_small_subspaces_type_id + 1); @@ -42,29 +42,4 @@ TensorBufferTypeMapper::TensorBufferTypeMapper(uint32_t max_small_subspaces_type TensorBufferTypeMapper::~TensorBufferTypeMapper() = default; -uint32_t -TensorBufferTypeMapper::get_type_id(size_t array_size) const -{ - assert(!_array_sizes.empty()); - auto result = std::lower_bound(_array_sizes.begin() + 1, _array_sizes.end(), array_size); - if (result == _array_sizes.end()) { - return 0; // type id 0 uses LargeSubspacesBufferType - } - return result - _array_sizes.begin(); -} - -size_t -TensorBufferTypeMapper::get_array_size(uint32_t type_id) const -{ - assert(type_id > 0 && type_id < _array_sizes.size()); - return _array_sizes[type_id]; -} - -uint32_t -TensorBufferTypeMapper::get_max_small_array_type_id(uint32_t max_small_array_type_id) const noexcept -{ - auto clamp_type_id = _array_sizes.size() - 1; - return (clamp_type_id < max_small_array_type_id) ? clamp_type_id : max_small_array_type_id; -} - } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.h b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.h index ad2116a429c..422224751e3 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.h @@ -2,9 +2,7 @@ #pragma once -#include <cstddef> -#include <cstdint> -#include <vector> +#include <vespa/vespalib/datastore/array_store_type_mapper.h> namespace search::tensor { @@ -16,9 +14,8 @@ class TensorBufferOperations; * This class provides mapping between type ids and array sizes needed for * storing a tensor. */ -class TensorBufferTypeMapper +class TensorBufferTypeMapper : public vespalib::datastore::ArrayStoreTypeMapper { - std::vector<size_t> _array_sizes; TensorBufferOperations* _ops; public: using SmallBufferType = SmallSubspacesBufferType; @@ -28,10 +25,7 @@ public: TensorBufferTypeMapper(uint32_t max_small_subspaces_type_id, double grow_factor, TensorBufferOperations* ops); ~TensorBufferTypeMapper(); - uint32_t get_type_id(size_t array_size) const; - size_t get_array_size(uint32_t type_id) const; TensorBufferOperations& get_tensor_buffer_operations() const noexcept { return *_ops; } - uint32_t get_max_small_array_type_id(uint32_t max_small_array_type_id) const noexcept; }; } diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp index e3cf067bd49..6bba9d96d02 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp @@ -5,6 +5,7 @@ #include "bitdecode64.h" #include "fpfactory.h" #include <vespa/searchlib/queryeval/iterators.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".searchlib.test.fake_eg_compr64_filter_occ"); diff --git a/searchlib/src/vespa/searchlib/test/features/CMakeLists.txt b/searchlib/src/vespa/searchlib/test/features/CMakeLists.txt index ba70fe57e88..f51ccea5678 100644 --- a/searchlib/src/vespa/searchlib/test/features/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/test/features/CMakeLists.txt @@ -4,4 +4,5 @@ vespa_add_library(searchlib_searchlib_test_features distance_closeness_fixture.cpp DEPENDS searchlib + GTest::GTest ) diff --git a/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.cpp b/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.cpp index e161a4e9839..e0444e8dca7 100644 --- a/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.cpp +++ b/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.cpp @@ -7,6 +7,8 @@ #include <vespa/eval/eval/value_type.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> +#include <vespa/searchlib/tensor/direct_tensor_attribute.h> +#include <vespa/searchlib/tensor/serialized_fast_value_attribute.h> using search::attribute::BasicType; using search::attribute::CollectionType; @@ -15,6 +17,9 @@ using search::attribute::DistanceMetric; using search::fef::test::IndexEnvironment; using search::fef::test::QueryEnvironment; using search::tensor::DenseTensorAttribute; +using search::tensor::DirectTensorAttribute; +using search::tensor::SerializedFastValueAttribute; +using search::tensor::TensorAttribute; using vespalib::eval::SimpleValue; using vespalib::eval::TensorSpec; using vespalib::eval::Value; @@ -24,15 +29,23 @@ namespace search::features::test { namespace { -std::shared_ptr<DenseTensorAttribute> +std::shared_ptr<TensorAttribute> create_tensor_attribute(const vespalib::string& attr_name, const vespalib::string& tensor_type, + bool direct_tensor, uint32_t docid_limit) { Config cfg(BasicType::TENSOR, CollectionType::SINGLE); cfg.setTensorType(ValueType::from_spec(tensor_type)); cfg.set_distance_metric(DistanceMetric::Euclidean); - auto result = std::make_shared<DenseTensorAttribute>(attr_name, cfg); + std::shared_ptr<TensorAttribute> result; + if (cfg.tensorType().is_dense()) { + result = std::make_shared<DenseTensorAttribute>(attr_name, cfg); + } else if (direct_tensor) { + result = std::make_shared<DirectTensorAttribute>(attr_name, cfg); + } else { + result = std::make_shared<SerializedFastValueAttribute>(attr_name, cfg); + } result->addReservedDoc(); result->addDocs(docid_limit-1); result->commit(); @@ -47,10 +60,21 @@ DistanceClosenessFixture::DistanceClosenessFixture(size_t fooCnt, size_t barCnt, const Labels& labels, const vespalib::string& featureName, const vespalib::string& query_tensor) + : DistanceClosenessFixture("tensor(x[2])", false, fooCnt, barCnt, labels, featureName, query_tensor) +{ +} + +DistanceClosenessFixture::DistanceClosenessFixture(const vespalib::string& tensor_type, + bool direct_tensor, + size_t fooCnt, size_t barCnt, + const Labels& labels, + const vespalib::string& featureName, + const vespalib::string& query_tensor) : queryEnv(&indexEnv), rankSetup(factory, indexEnv), mdl(), match_data(), rankProgram(), fooHandles(), barHandles(), tensor_attr(), - docid_limit(11) + docid_limit(11), + _failed(false) { for (size_t i = 0; i < fooCnt; ++i) { uint32_t fieldId = indexEnv.getFieldByName("foo")->id(); @@ -72,14 +96,18 @@ DistanceClosenessFixture::DistanceClosenessFixture(size_t fooCnt, size_t barCnt, queryEnv.getTerms().push_back(term); } if (!query_tensor.empty()) { - tensor_attr = create_tensor_attribute("bar", "tensor(x[2])", docid_limit); + tensor_attr = create_tensor_attribute("bar", tensor_type, direct_tensor, docid_limit); indexEnv.getAttributeMap().add(tensor_attr); + search::fef::indexproperties::type::Attribute::set(indexEnv.getProperties(), "bar", tensor_type); set_query_tensor("qbar", "tensor(x[2])", TensorSpec::from_expr(query_tensor)); } labels.inject(queryEnv.getProperties()); rankSetup.setFirstPhaseRank(featureName); rankSetup.setIgnoreDefaultRankFeatures(true); - ASSERT_TRUE(rankSetup.compile()); + EXPECT_TRUE(rankSetup.compile()) << (_failed = true, ""); + if (_failed) { + return; + } rankSetup.prepareSharedState(queryEnv, queryEnv.getObjectStore()); match_data = mdl.createMatchData(); rankProgram = rankSetup.create_first_phase_program(); diff --git a/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.h b/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.h index cc1c0a6fb15..8aae1ecb942 100644 --- a/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.h +++ b/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.h @@ -8,12 +8,12 @@ #include <vespa/searchlib/fef/test/indexenvironmentbuilder.h> #include <vespa/searchlib/fef/test/labels.h> #include <vespa/searchlib/fef/test/queryenvironment.h> -#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/gtest/gtest.h> using namespace search::fef; using namespace search::fef::test; -namespace search::tensor { class DenseTensorAttribute; } +namespace search::tensor { class TensorAttribute; } namespace vespalib::eval { class TensorSpec; } namespace search::features::test { @@ -33,12 +33,13 @@ struct IndexEnvironmentFixture { IndexEnvironmentBuilder builder(indexEnv); builder.addField(FieldType::ATTRIBUTE, FieldInfo::CollectionType::SINGLE, FieldInfo::DataType::INT64, "foo"); builder.addField(FieldType::ATTRIBUTE, FieldInfo::CollectionType::SINGLE, FieldInfo::DataType::TENSOR, "bar"); + builder.addField(FieldType::INDEX, FieldInfo::CollectionType::SINGLE, FieldInfo::DataType::TENSOR, "ibar"); } }; struct FeatureDumpFixture : public IDumpFeatureVisitor { virtual void visitDumpFeature(const vespalib::string &) override { - TEST_ERROR("no features should be dumped"); + FAIL() << "no features should be dumped"; } FeatureDumpFixture() : IDumpFeatureVisitor() {} ~FeatureDumpFixture() override; @@ -55,11 +56,17 @@ struct DistanceClosenessFixture : BlueprintFactoryFixture, IndexEnvironmentFixtu RankProgram::UP rankProgram; std::vector<TermFieldHandle> fooHandles; std::vector<TermFieldHandle> barHandles; - std::shared_ptr<search::tensor::DenseTensorAttribute> tensor_attr; + std::shared_ptr<search::tensor::TensorAttribute> tensor_attr; uint32_t docid_limit; + bool _failed; DistanceClosenessFixture(size_t fooCnt, size_t barCnt, const Labels &labels, const vespalib::string &featureName, const vespalib::string& query_tensor = ""); + DistanceClosenessFixture(const vespalib::string& tensor_type, + bool direct_tensor, + size_t fooCnt, size_t barCnt, + const Labels &labels, const vespalib::string &featureName, + const vespalib::string& query_tensor = ""); ~DistanceClosenessFixture(); void set_attribute_tensor(uint32_t docid, const vespalib::eval::TensorSpec& spec); void set_query_tensor(const vespalib::string& query_tensor_name, @@ -68,17 +75,21 @@ struct DistanceClosenessFixture : BlueprintFactoryFixture, IndexEnvironmentFixtu feature_t getScore(uint32_t docId) { return Utils::getScoreFeature(*rankProgram, docId); } + vespalib::eval::Value::CREF getObject(uint32_t docId) { + return Utils::getObjectFeature(*rankProgram, docId); + } void setScore(TermFieldHandle handle, uint32_t docId, feature_t score) { match_data->resolveTermField(handle)->setRawScore(docId, score); } void setFooScore(uint32_t i, uint32_t docId, feature_t distance) { - ASSERT_LESS(i, fooHandles.size()); + ASSERT_LT(i, fooHandles.size()); setScore(fooHandles[i], docId, 1.0/(1.0+distance)); } void setBarScore(uint32_t i, uint32_t docId, feature_t distance) { - ASSERT_LESS(i, barHandles.size()); + ASSERT_LT(i, barHandles.size()); setScore(barHandles[i], docId, 1.0/(1.0+distance)); } + bool failed() const noexcept { return _failed; } }; } diff --git a/searchlib/src/vespa/searchlib/transactionlog/client_session.cpp b/searchlib/src/vespa/searchlib/transactionlog/client_session.cpp index c44edfcec86..f5cc3c0b247 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/client_session.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/client_session.cpp @@ -54,7 +54,7 @@ Session::commit(const vespalib::ConstBufferRef & buf) int retcode = _tlc.rpc(req); retval = (retcode == 0); if (retval) { - req->SubRef(); + req->internal_subref(); } else { vespalib::string msg; if (req->GetReturn() != nullptr) { @@ -62,7 +62,7 @@ Session::commit(const vespalib::ConstBufferRef & buf) } else { msg = vespalib::make_string("Clientside error %s: error(%d): %s", req->GetMethodName(), req->GetErrorCode(), req->GetErrorMessage()); } - req->SubRef(); + req->internal_subref(); throw std::runtime_error(vespalib::make_string("commit failed with code %d. server says: %s", retcode, msg.c_str())); } } @@ -81,7 +81,7 @@ Session::status(SerialNum & b, SerialNum & e, size_t & count) e = req->GetReturn()->GetValue(2)._intval64; count = req->GetReturn()->GetValue(3)._intval64; } - req->SubRef(); + req->internal_subref(); return (retval == 0); } @@ -93,7 +93,7 @@ Session::erase(const SerialNum & to) req->GetParams()->AddString(_domain.c_str()); req->GetParams()->AddInt64(to); int32_t retval(_tlc.rpc(req)); - req->SubRef(); + req->internal_subref(); if (retval == 1) { LOG(warning, "Prune to %" PRIu64 " denied since there were active visitors in that area", to); } @@ -113,7 +113,7 @@ Session::sync(const SerialNum &syncTo, SerialNum &syncedTo) if (retval == 0) { syncedTo = req->GetReturn()->GetValue(1)._intval64; } - req->SubRef(); + req->internal_subref(); return (retval == 0); } @@ -138,7 +138,7 @@ bool Session::init(FRT_RPCRequest *req) { int32_t retval(_tlc.rpc(req)); - req->SubRef(); + req->internal_subref(); if (retval > 0) { clear(); _sessionId = retval; @@ -171,7 +171,7 @@ Session::run() req->GetParams()->AddString(_domain.c_str()); req->GetParams()->AddInt32(_sessionId); int32_t retval(_tlc.rpc(req)); - req->SubRef(); + req->internal_subref(); return (retval == 0); } @@ -188,7 +188,7 @@ Session::close() if ( (retval = _tlc.rpc(req)) > 0) { std::this_thread::sleep_for(10ms); } - req->SubRef(); + req->internal_subref(); } while ( retval == 1 ); } return (retval == 0); diff --git a/searchlib/src/vespa/searchlib/transactionlog/session.cpp b/searchlib/src/vespa/searchlib/transactionlog/session.cpp index b11f4027ae7..ec77a5f150e 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/session.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/session.cpp @@ -4,6 +4,7 @@ #include "domainpart.h" #include <vespa/fastlib/io/bufferedfile.h> #include <cassert> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".transactionlog.session"); diff --git a/searchlib/src/vespa/searchlib/transactionlog/trans_log_server_explorer.cpp b/searchlib/src/vespa/searchlib/transactionlog/trans_log_server_explorer.cpp index 103438d6d61..f49cef72172 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/trans_log_server_explorer.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/trans_log_server_explorer.cpp @@ -37,7 +37,7 @@ struct DomainExplorer : vespalib::StateExplorer { { FastOS_StatInfo stat_info; FastOS_File::Stat(part_in.file.c_str(), &stat_info); - part.setString("lastModified", vespalib::to_string(vespalib::system_time(std::chrono::duration_cast<vespalib::system_time::duration>(std::chrono::nanoseconds(stat_info._modifiedTimeNS))))); + part.setString("lastModified", vespalib::to_string(stat_info._modifiedTime)); } } } diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogclient.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogclient.cpp index 133fabd3e5f..8916a4cf0b5 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogclient.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogclient.cpp @@ -8,7 +8,6 @@ #include <vespa/fnet/transport.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/fastos/thread.h> #include <vespa/log/log.h> @@ -81,7 +80,7 @@ void TransLogClient::disconnect() { if (_target) { - _target->SubRef(); + _target->internal_subref(); } } @@ -92,7 +91,7 @@ TransLogClient::create(const vespalib::string & domain) req->SetMethodName("createDomain"); req->GetParams()->AddString(domain.c_str()); int32_t retval(rpc(req)); - req->SubRef(); + req->internal_subref(); return (retval == 0); } @@ -103,7 +102,7 @@ TransLogClient::remove(const vespalib::string & domain) req->SetMethodName("deleteDomain"); req->GetParams()->AddString(domain.c_str()); int32_t retval(rpc(req)); - req->SubRef(); + req->internal_subref(); return (retval == 0); } @@ -114,7 +113,7 @@ TransLogClient::open(const vespalib::string & domain) req->SetMethodName("openDomain"); req->GetParams()->AddString(domain.c_str()); int32_t retval(rpc(req)); - req->SubRef(); + req->internal_subref(); if (retval == 0) { return std::make_unique<Session>(domain, *this); } @@ -139,7 +138,7 @@ TransLogClient::listDomains(std::vector<vespalib::string> & dir) dir.push_back(d); } } - req->SubRef(); + req->internal_subref(); return (retval == 0); } diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogclient.h b/searchlib/src/vespa/searchlib/transactionlog/translogclient.h index c3dcecf93b3..72a55f6ae77 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogclient.h +++ b/searchlib/src/vespa/searchlib/transactionlog/translogclient.h @@ -12,7 +12,6 @@ class FNET_Transport; class FRT_Supervisor; class FRT_Target; -class FastOS_ThreadPool; namespace vespalib { class ThreadStackExecutorBase; } namespace search::transactionlog::client { diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index c96b0cdcd61..db02f4f037e 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -361,9 +361,9 @@ public: RPCDestination(FRT_Supervisor & supervisor, FNET_Connection * connection) : _supervisor(supervisor), _connection(connection), _ok(true) { - _connection->AddRef(); + _connection->internal_addref(); } - ~RPCDestination() override { _connection->SubRef(); } + ~RPCDestination() override { _connection->internal_subref(); } bool ok() const override { return _ok; @@ -395,7 +395,7 @@ private: if ( ! ((retval == client::RPC::OK) || (retval == FRTE_RPC_CONNECTION)) ) { LOG(error, "Return value != OK(%d) in send for method 'visitCallback'.", retval); } - req->SubRef(); + req->internal_subref(); return (retval == client::RPC::OK); } diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.h b/searchlib/src/vespa/searchlib/transactionlog/translogserver.h index 2c5fbf51a08..83993da5964 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.h +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.h @@ -7,6 +7,7 @@ #include <vespa/fnet/frt/invokable.h> #include <shared_mutex> #include <atomic> +#include <thread> class FRT_Supervisor; class FNET_Transport; diff --git a/searchlib/src/vespa/searchlib/util/filekit.cpp b/searchlib/src/vespa/searchlib/util/filekit.cpp index 68557159635..07eab9bb2be 100644 --- a/searchlib/src/vespa/searchlib/util/filekit.cpp +++ b/searchlib/src/vespa/searchlib/util/filekit.cpp @@ -95,7 +95,7 @@ FileKit::getModificationTime(const vespalib::string &name) { FastOS_StatInfo statInfo; if (FastOS_File::Stat(name.c_str(), &statInfo)) { - return vespalib::system_time(std::chrono::duration_cast<vespalib::system_time::duration>(std::chrono::nanoseconds(statInfo._modifiedTimeNS))); + return statInfo._modifiedTime; } return vespalib::system_time(); } diff --git a/searchlib/src/vespa/searchlib/util/filesizecalculator.cpp b/searchlib/src/vespa/searchlib/util/filesizecalculator.cpp index 16a96c7b439..c50d402db0e 100644 --- a/searchlib/src/vespa/searchlib/util/filesizecalculator.cpp +++ b/searchlib/src/vespa/searchlib/util/filesizecalculator.cpp @@ -2,6 +2,7 @@ #include "filesizecalculator.h" #include <vespa/vespalib/data/fileheader.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".searchlib.util.filesizecalculator"); diff --git a/searchlib/src/vespa/searchlib/util/runnable.h b/searchlib/src/vespa/searchlib/util/runnable.h deleted file mode 100644 index 4b353209762..00000000000 --- a/searchlib/src/vespa/searchlib/util/runnable.h +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <mutex> -#include <condition_variable> -#include <vespa/fastos/thread.h> - -namespace search { - -class Runnable : public FastOS_Runnable -{ -protected: - std::mutex _lock; - std::condition_variable _cond; - bool _done; - bool _stopped; - -public: - Runnable() : - _lock(), _cond(), _done(false), _stopped(false) - { } - void Run(FastOS_ThreadInterface *, void *) override { - doRun(); - - std::lock_guard<std::mutex> guard(_lock); - _stopped = true; - _cond.notify_all(); - } - virtual void doRun() = 0; - void stop() { - std::lock_guard<std::mutex> guard(_lock); - _done = true; - } - void join() { - std::unique_lock<std::mutex> guard(_lock); - while (!_stopped) { - _cond.wait(guard); - } - } -}; - -} // search - diff --git a/searchlib/src/vespa/searchlib/util/url.cpp b/searchlib/src/vespa/searchlib/util/url.cpp index 5eeb16e3f3b..141f54363e9 100644 --- a/searchlib/src/vespa/searchlib/util/url.cpp +++ b/searchlib/src/vespa/searchlib/util/url.cpp @@ -3,6 +3,7 @@ #include "url.h" #include <algorithm> #include <cstdio> +#include <cstring> #include <vespa/log/log.h> LOG_SETUP(".searchlib.util.url"); diff --git a/searchsummary/src/tests/juniper/auxTest.cpp b/searchsummary/src/tests/juniper/auxTest.cpp index 8eda6a2132a..a43294e709f 100644 --- a/searchsummary/src/tests/juniper/auxTest.cpp +++ b/searchsummary/src/tests/juniper/auxTest.cpp @@ -11,9 +11,7 @@ LOG_SETUP(".auxtest"); #define COLOR_HIGH_ON "\e[1;31m" #define COLOR_HIGH_OFF "\e[0m" -#ifndef FASTOS_DEBUG static int debug_level = 0; -#endif bool color_highlight = false; bool verbose = false; diff --git a/searchsummary/src/tests/juniper/testenv.cpp b/searchsummary/src/tests/juniper/testenv.cpp index cc6a6458376..1f8f52d8766 100644 --- a/searchsummary/src/tests/juniper/testenv.cpp +++ b/searchsummary/src/tests/juniper/testenv.cpp @@ -26,11 +26,7 @@ TestEnv::TestEnv(int argc, char **argv, const char* propfile) : switch (c) { case 'd': -#ifdef FASTOS_DEBUG - debug_level = strtol(optarg, NULL, 0); -#else fprintf(stderr, "This version of Juniper compiled without debug\n"); -#endif break; case 'c': color_highlight = true; diff --git a/searchsummary/src/vespa/juniper/Matcher.cpp b/searchsummary/src/vespa/juniper/Matcher.cpp index 22d1bbc7e96..73362aac5a3 100644 --- a/searchsummary/src/vespa/juniper/Matcher.cpp +++ b/searchsummary/src/vespa/juniper/Matcher.cpp @@ -1,7 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <algorithm> -#include <string> #include "query.h" #include "juniperdebug.h" #include "sumdesc.h" @@ -10,6 +8,10 @@ #include "juniperparams.h" #include "config.h" #include <sstream> +#include <algorithm> +#include <string> +#include <cinttypes> + #include <vespa/log/log.h> LOG_SETUP(".juniper.matcher"); diff --git a/searchsummary/src/vespa/juniper/juniperdebug.h b/searchsummary/src/vespa/juniper/juniperdebug.h index d475134dae3..f6d4eda2c46 100644 --- a/searchsummary/src/vespa/juniper/juniperdebug.h +++ b/searchsummary/src/vespa/juniper/juniperdebug.h @@ -27,23 +27,10 @@ /* Logging to log object (juniperlog summary field) */ #define JL(level, stmt) do { if (_log_mask & level) { stmt; } } while (0) -#ifdef FASTOS_DEBUG -extern unsigned debug_level; -#define JD(level, stmt) do { if (debug_level & level) { stmt; } } while (0) -# warning "FASTOS_DEBUG is defined" - -/* Invariant checking */ - -#define JD_INVAR(level, condition, action, log) \ - do { if (!(condition)) { if (debug_level & level) { log; } action; } } while (0) -#else - #define JD_INVAR(level, condition, action, log) \ do { if (!(condition)) { action; } } while (0) #define JD(level, stmt) -#endif - template <class _container> void dump_list(_container& __c) { diff --git a/searchsummary/src/vespa/juniper/rpinterface.cpp b/searchsummary/src/vespa/juniper/rpinterface.cpp index 202b96a442d..afda82c1110 100644 --- a/searchsummary/src/vespa/juniper/rpinterface.cpp +++ b/searchsummary/src/vespa/juniper/rpinterface.cpp @@ -26,13 +26,6 @@ bool AnalyseCompatible(Config* conf1, Config* conf2) void SetDebug(unsigned int mask) { -#ifdef FASTOS_DEBUG - if (mask & ~1 && mask != debug_level) - LOG(info, "Juniper debug mode enabled (0x%x)", mask); - else if (! (debug_level & ~1)) - LOG(info, "Juniper debug mode disabled (0x%x)", mask); - debug_level = mask; -#else // Make sure we do not get 200 of these warnings per query.. static bool warning_printed = false; if (mask && !warning_printed) @@ -41,7 +34,6 @@ void SetDebug(unsigned int mask) "Juniper debug mode requested in binary compiled without debug support!"); warning_printed = true; } -#endif } diff --git a/searchsummary/src/vespa/juniper/sumdesc.cpp b/searchsummary/src/vespa/juniper/sumdesc.cpp index 18e1b7bbd11..aa6aededa0c 100644 --- a/searchsummary/src/vespa/juniper/sumdesc.cpp +++ b/searchsummary/src/vespa/juniper/sumdesc.cpp @@ -6,6 +6,7 @@ #include "Matcher.h" #include "appender.h" #include <vespa/fastlib/text/unicodeutil.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".juniper.sumdesc"); diff --git a/searchsummary/src/vespa/juniper/tokenizer.cpp b/searchsummary/src/vespa/juniper/tokenizer.cpp index 9253f81cf25..965befe01e3 100644 --- a/searchsummary/src/vespa/juniper/tokenizer.cpp +++ b/searchsummary/src/vespa/juniper/tokenizer.cpp @@ -3,6 +3,7 @@ #include "tokenizer.h" #include "juniperdebug.h" #include <vespa/fastlib/text/wordfolder.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".juniper.tokenizer"); @@ -40,7 +41,6 @@ void JuniperTokenizer::scan() { if (_registry == NULL) { // explicit prefetching seems to have negative effect with many threads - // FastOS_Prefetch::NT(const_cast<void *>((const void *)(src + 32))); src = _wordfolder->UCS4Tokenize(src, src_end, dst, dst_end, startpos, result_len); } else { const char * tmpSrc = _registry->tokenize(src, src_end, dst, dst_end, startpos, result_len); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp index 74d67aabe88..e606c6f08bb 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp @@ -97,8 +97,8 @@ SingleAttrDFW::insertField(uint32_t docid, GetDocsumsState& state, Inserter &tar break; } case BasicType::STRING: { - const char *s = v.getString(docid, nullptr, 0); // no need to pass in a buffer, this attribute has a string storage. - target.insertString(vespalib::Memory(s)); + auto s = v.get_raw(docid); + target.insertString(vespalib::Memory(s.data(), s.size())); break; } case BasicType::REFERENCE: diff --git a/security-utils/src/main/java/com/yahoo/security/tls/Capability.java b/security-utils/src/main/java/com/yahoo/security/tls/Capability.java index 6a6471aa8ac..dce40681b90 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/Capability.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/Capability.java @@ -30,6 +30,7 @@ public enum Capability implements ToCapabilitySet { CONTENT__METRICS_API("vespa.content.metrics_api"), CONTENT__PROTON_ADMIN_API("vespa.content.proton_admin_api"), CONTENT__SEARCH_API("vespa.content.search_api"), + CONTENT__STATE_API("vespa.content.state_api"), CONTENT__STATUS_PAGES("vespa.content.status_pages"), CONTENT__STORAGE_API("vespa.content.storage_api"), LOGSERVER_API("vespa.logserver.api"), diff --git a/security-utils/src/main/java/com/yahoo/security/tls/CapabilitySet.java b/security-utils/src/main/java/com/yahoo/security/tls/CapabilitySet.java index 8fa077027a9..197088ff434 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/CapabilitySet.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/CapabilitySet.java @@ -1,17 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.security.tls; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -30,13 +29,14 @@ public class CapabilitySet implements ToCapabilitySet { "vespa.all", Capability.values()); public static final CapabilitySet TELEMETRY = predefined( "vespa.telemetry", - Capability.CONTENT__STATUS_PAGES, Capability.CONTENT__METRICS_API, Capability.CONTAINER__STATE_API, - Capability.METRICSPROXY__METRICS_API, Capability.SENTINEL__CONNECTIVITY_CHECK); + Capability.CONTENT__STATUS_PAGES, Capability.CONTENT__STATE_API, Capability.CONTENT__METRICS_API, + Capability.CONTAINER__STATE_API, Capability.METRICSPROXY__METRICS_API, + Capability.SENTINEL__CONNECTIVITY_CHECK); - private static final CapabilitySet SHARED_CAPABILITIES_APP_NODE = CapabilitySet.of( + private static final CapabilitySet SHARED_CAPABILITIES_APP_NODE = CapabilitySet.unionOf(List.of( Capability.LOGSERVER_API, Capability.CONFIGSERVER__CONFIG_API, Capability.CONFIGSERVER__FILEDISTRIBUTION_API, Capability.CONFIGPROXY__CONFIG_API, - Capability.CONFIGPROXY__FILEDISTRIBUTION_API, Capability.SLOBROK__API, TELEMETRY); + Capability.CONFIGPROXY__FILEDISTRIBUTION_API, Capability.SLOBROK__API, TELEMETRY)); public static final CapabilitySet CONTENT_NODE = predefined( "vespa.content_node", @@ -44,7 +44,8 @@ public class CapabilitySet implements ToCapabilitySet { SHARED_CAPABILITIES_APP_NODE); public static final CapabilitySet CONTAINER_NODE = predefined( "vespa.container_node", - Capability.CONTENT__DOCUMENT_API, Capability.CONTENT__SEARCH_API, SHARED_CAPABILITIES_APP_NODE); + Capability.CONTAINER__DOCUMENT_API, Capability.CONTENT__DOCUMENT_API, Capability.CONTENT__SEARCH_API, + SHARED_CAPABILITIES_APP_NODE); public static final CapabilitySet CLUSTER_CONTROLLER_NODE = predefined( "vespa.cluster_controller_node", Capability.CONTENT__CLUSTER_CONTROLLER__INTERNAL_STATE_API, @@ -58,7 +59,7 @@ public class CapabilitySet implements ToCapabilitySet { TELEMETRY); private static CapabilitySet predefined(String name, ToCapabilitySet... capabilities) { - var instance = CapabilitySet.of(capabilities); + var instance = CapabilitySet.unionOf(List.of(capabilities)); PREDEFINED.put(name, instance); return instance; } @@ -84,14 +85,14 @@ public class CapabilitySet implements ToCapabilitySet { return new CapabilitySet(caps); } - public static CapabilitySet unionOf(Collection<CapabilitySet> capSets) { + public static CapabilitySet ofSets(Collection<CapabilitySet> capSets) { EnumSet<Capability> union = EnumSet.noneOf(Capability.class); capSets.forEach(cs -> union.addAll(cs.caps)); return new CapabilitySet(union); } - public static CapabilitySet of(ToCapabilitySet... capabilities) { - return CapabilitySet.unionOf(Arrays.stream(capabilities).map(ToCapabilitySet::toCapabilitySet).toList()); + public static CapabilitySet unionOf(Collection<ToCapabilitySet> caps) { + return CapabilitySet.ofSets(caps.stream().map(ToCapabilitySet::toCapabilitySet).toList()); } public static CapabilitySet of(EnumSet<Capability> caps) { return new CapabilitySet(EnumSet.copyOf(caps)); } @@ -106,8 +107,33 @@ public class CapabilitySet implements ToCapabilitySet { public boolean has(Collection<Capability> caps) { return this.caps.containsAll(caps); } public boolean has(Capability... caps) { return this.caps.containsAll(List.of(caps)); } - public SortedSet<String> toNames() { - return caps.stream().map(Capability::asString).collect(Collectors.toCollection(TreeSet::new)); + public Set<String> toCapabilityNames() { + return caps.stream().map(Capability::asString).collect(Collectors.toSet()); + } + + /** return name of the capability set if predefined, otherwise names of the individual capabilities */ + public Set<String> resolveNames() { + var predefinedName = toPredefinedName().orElse(null); + if (predefinedName != null) return Set.of(predefinedName); + return toCapabilityNames(); + } + + /** @return the name if this is a predefined capability set, or empty if not */ + public Optional<String> toPredefinedName() { + return PREDEFINED.entrySet().stream() + .filter(e -> e.getValue().equals(this)) + .map(Map.Entry::getKey) + .findFirst(); + } + + public static Set<String> resolveNames(Collection<ToCapabilitySet> capabilities) { + var names = new HashSet<String>(); + for (ToCapabilitySet tcs : capabilities) { + if (tcs instanceof Capability c) names.add(c.asString()); + else if (tcs instanceof CapabilitySet cs) names.addAll(cs.resolveNames()); + else throw new IllegalArgumentException(tcs.toString()); + } + return Set.copyOf(names); } public Set<Capability> asSet() { return Collections.unmodifiableSet(caps); } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/ConnectionAuthContext.java b/security-utils/src/main/java/com/yahoo/security/tls/ConnectionAuthContext.java index d7ea93955af..9252b5619f9 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/ConnectionAuthContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/ConnectionAuthContext.java @@ -8,6 +8,7 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.logging.Logger; +import java.util.stream.Collectors; import static com.yahoo.security.SubjectAlternativeName.Type.DNS; import static com.yahoo.security.SubjectAlternativeName.Type.URI; @@ -78,10 +79,14 @@ public record ConnectionAuthContext(List<X509Certificate> peerCertificateChain, b.append(". Peer "); if (peer != null) b.append("'").append(peer).append("' "); return b.append("with ").append(peerCertificateString().orElse("<missing-certificate>")).append(". Requires capabilities ") - .append(required.toNames()).append(" but peer has ").append(capabilities.toNames()) + .append(toCapabilityNames(required)).append(" but peer has ").append(toCapabilityNames(capabilities)) .append(".").toString(); } + private static String toCapabilityNames(CapabilitySet capabilities) { + return capabilities.toCapabilityNames().stream().sorted().collect(Collectors.joining(", ", "[", "]")); + } + public Optional<X509Certificate> peerCertificate() { return peerCertificateChain.isEmpty() ? Optional.empty() : Optional.of(peerCertificateChain.get(0)); } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/PeerAuthorizer.java b/security-utils/src/main/java/com/yahoo/security/tls/PeerAuthorizer.java index 746fce0e290..d0e1a33fcac 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/PeerAuthorizer.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/PeerAuthorizer.java @@ -49,7 +49,7 @@ public class PeerAuthorizer { // TODO Pass this through constructor CapabilityMode capabilityMode = TransportSecurityUtils.getCapabilityMode(); return new ConnectionAuthContext( - certChain, CapabilitySet.unionOf(grantedCapabilities), matchedPolicies, capabilityMode); + certChain, CapabilitySet.ofSets(grantedCapabilities), matchedPolicies, capabilityMode); } private static boolean matchesPolicy(PeerPolicy peerPolicy, String cn, List<String> sans) { diff --git a/security-utils/src/main/java/com/yahoo/security/tls/PeerPolicy.java b/security-utils/src/main/java/com/yahoo/security/tls/PeerPolicy.java index ea3d4cfe002..f713bcb0b08 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/PeerPolicy.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/PeerPolicy.java @@ -1,17 +1,25 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.security.tls; +import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.Set; /** * @author bjorncs */ -public record PeerPolicy(String policyName, Optional<String> description, CapabilitySet capabilities, - List<RequiredPeerCredential> requiredCredentials) { +public record PeerPolicy(String policyName, Optional<String> description, Set<String> capabilityNames, + CapabilitySet capabilities, List<RequiredPeerCredential> requiredCredentials) { public PeerPolicy { requiredCredentials = List.copyOf(requiredCredentials); + capabilityNames = Set.copyOf(capabilityNames); + } + + public PeerPolicy(String policyName, Optional<String> description, + CapabilitySet capabilities, List<RequiredPeerCredential> requiredCredentials) { + this(policyName, description, capabilities.resolveNames(), capabilities, requiredCredentials); } public PeerPolicy(String policyName, List<RequiredPeerCredential> requiredCredentials) { @@ -21,4 +29,16 @@ public record PeerPolicy(String policyName, Optional<String> description, Capabi public PeerPolicy(String policyName, String description, List<RequiredPeerCredential> requiredCredentials) { this(policyName, Optional.ofNullable(description), CapabilitySet.all(), requiredCredentials); } + + public PeerPolicy(String policyName, Optional<String> description, Collection<ToCapabilitySet> capabilities, + List<RequiredPeerCredential> requiredCredentials) { + this(policyName, description, CapabilitySet.resolveNames(capabilities), + CapabilitySet.unionOf(capabilities), requiredCredentials); + } + + public PeerPolicy(String policyName, Optional<String> description, Set<String> capabilities, + List<RequiredPeerCredential> requiredCredentials) { + this(policyName, description, capabilities, CapabilitySet.fromNames(capabilities), + requiredCredentials); + } } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptionsJsonSerializer.java b/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptionsJsonSerializer.java index 34626e23e7a..66b90b32f79 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptionsJsonSerializer.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptionsJsonSerializer.java @@ -96,15 +96,15 @@ class TransportSecurityOptionsJsonSerializer { throw missingFieldException("required-credentials"); } return new PeerPolicy(authorizedPeer.name, Optional.ofNullable(authorizedPeer.description), - toCapabilities(authorizedPeer.capabilities), toRequestPeerCredentials(authorizedPeer.requiredCredentials)); + toCapabilities(authorizedPeer.capabilities), toRequestPeerCredentials(authorizedPeer.requiredCredentials)); } - private static CapabilitySet toCapabilities(List<String> capabilities) { - if (capabilities == null) return CapabilitySet.all(); + private static Set<String> toCapabilities(List<String> capabilities) { + if (capabilities == null) return Set.of(CapabilitySet.ALL.toPredefinedName().get()); if (capabilities.isEmpty()) throw new IllegalArgumentException("\"capabilities\" array must either be not present " + "(implies all capabilities) or contain at least one capability name"); - return CapabilitySet.fromNames(capabilities); + return Set.copyOf(capabilities); } private static List<RequiredPeerCredential> toRequestPeerCredentials(List<RequiredCredential> requiredCredentials) { @@ -148,7 +148,7 @@ class TransportSecurityOptionsJsonSerializer { authorizedPeer.description = peerPolicy.description().orElse(null); CapabilitySet caps = peerPolicy.capabilities(); if (!caps.hasAll()) { - authorizedPeer.capabilities = List.copyOf(caps.toNames()); + authorizedPeer.capabilities = peerPolicy.capabilityNames().stream().sorted().toList(); } for (RequiredPeerCredential requiredPeerCredential : peerPolicy.requiredCredentials()) { RequiredCredential requiredCredential = new RequiredCredential(); diff --git a/security-utils/src/test/java/com/yahoo/security/tls/CapabilitySetTest.java b/security-utils/src/test/java/com/yahoo/security/tls/CapabilitySetTest.java index 87b16dbff1f..3fa75df27e1 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/CapabilitySetTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/CapabilitySetTest.java @@ -4,8 +4,6 @@ package com.yahoo.security.tls; import org.junit.jupiter.api.Test; import java.util.Arrays; -import java.util.SortedSet; -import java.util.TreeSet; import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -17,10 +15,10 @@ class CapabilitySetTest { @Test void contains_all_capabilities() { - SortedSet<String> expectedNames = Arrays.stream(Capability.values()) + var expectedNames = Arrays.stream(Capability.values()) .map(Capability::asString) - .collect(Collectors.toCollection(TreeSet::new)); - SortedSet<String> actualNames = CapabilitySet.all().toNames(); + .collect(Collectors.toSet()); + var actualNames = CapabilitySet.all().toCapabilityNames(); assertEquals(expectedNames, actualNames); } diff --git a/slobrok/CMakeLists.txt b/slobrok/CMakeLists.txt index 332e1e90282..64f728a7008 100644 --- a/slobrok/CMakeLists.txt +++ b/slobrok/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalib fnet configdefinitions diff --git a/slobrok/src/apps/check_slobrok/check_slobrok.cpp b/slobrok/src/apps/check_slobrok/check_slobrok.cpp index 1b69588a9fc..4e10c9ba6fe 100644 --- a/slobrok/src/apps/check_slobrok/check_slobrok.cpp +++ b/slobrok/src/apps/check_slobrok/check_slobrok.cpp @@ -55,7 +55,7 @@ void Slobrok_Checker::finiRPC() { if (_target != nullptr) { - _target->SubRef(); + _target->internal_subref(); _target = nullptr; } if (_server) { diff --git a/slobrok/src/apps/sbcmd/sbcmd.cpp b/slobrok/src/apps/sbcmd/sbcmd.cpp index 64c29cd92a9..51c8b81aa77 100644 --- a/slobrok/src/apps/sbcmd/sbcmd.cpp +++ b/slobrok/src/apps/sbcmd/sbcmd.cpp @@ -65,7 +65,7 @@ void Slobrok_CMD::finiRPC() { if (_target != nullptr) { - _target->SubRef(); + _target->internal_subref(); _target = nullptr; } if (_server) { @@ -181,7 +181,7 @@ Slobrok_CMD::main(int argc, char **argv) } } } - req->SubRef(); + req->internal_subref(); finiRPC(); return 0; } diff --git a/slobrok/src/tests/mirrorapi/mirrorapi.cpp b/slobrok/src/tests/mirrorapi/mirrorapi.cpp index 2ba54fdb9d0..6dac6b22b05 100644 --- a/slobrok/src/tests/mirrorapi/mirrorapi.cpp +++ b/slobrok/src/tests/mirrorapi/mirrorapi.cpp @@ -9,7 +9,6 @@ #include <vespa/fnet/frt/target.h> #include <vespa/fnet/transport.h> #include <thread> -#include <vespa/fastos/thread.h> #include <vespa/log/log.h> LOG_SETUP("mirrorapi_test"); @@ -67,8 +66,8 @@ Server::reg() FRT_Target *sb = _server.supervisor().GetTarget(_slobrokSpec.c_str()); sb->InvokeSync(req, 5.0); - sb->SubRef(); - req->SubRef(); + sb->internal_subref(); + req->internal_subref(); } @@ -139,12 +138,11 @@ Test::Main() cloud::config::SlobroksConfig::Slobrok slobrok; slobrok.connectionspec = "tcp/localhost:18501"; specBuilder.slobrok.push_back(slobrok); - FastOS_ThreadPool threadPool; FNET_Transport transport; FRT_Supervisor supervisor(&transport); MirrorAPI mirror(supervisor, slobrok::ConfiguratorFactory(config::ConfigUri::createFromInstance(specBuilder))); EXPECT_TRUE(!mirror.ready()); - transport.Start(&threadPool); + transport.Start(); std::this_thread::sleep_for(1s); a.reg(); diff --git a/slobrok/src/tests/rpc_mapping_monitor/rpc_mapping_monitor_test.cpp b/slobrok/src/tests/rpc_mapping_monitor/rpc_mapping_monitor_test.cpp index 13db95eb35c..0493a43adf3 100644 --- a/slobrok/src/tests/rpc_mapping_monitor/rpc_mapping_monitor_test.cpp +++ b/slobrok/src/tests/rpc_mapping_monitor/rpc_mapping_monitor_test.cpp @@ -21,11 +21,11 @@ struct Server : FRT_Invokable { FNET_Connection *last_conn; void set_last_conn(FNET_Connection *conn) { if (last_conn) { - last_conn->SubRef(); + last_conn->internal_subref(); } last_conn = conn; if (last_conn) { - last_conn->AddRef(); + last_conn->internal_addref(); } } Server(fnet::TimeTools::SP time_tools) @@ -210,7 +210,7 @@ TEST_F(RpcMappingMonitorTest, up_connection_is_reused) { a.last_conn = nullptr; EXPECT_TRUE(debugger.step_until([&]() { return (a.last_conn); })); EXPECT_EQ(a.last_conn, my_conn); - my_conn->SubRef(); + my_conn->internal_subref(); EXPECT_EQ(hist.map[foo_a].state(), State::UP); } diff --git a/slobrok/src/tests/standalone/standalone.cpp b/slobrok/src/tests/standalone/standalone.cpp index 33813fceb3a..653e4c64b0e 100644 --- a/slobrok/src/tests/standalone/standalone.cpp +++ b/slobrok/src/tests/standalone/standalone.cpp @@ -77,7 +77,7 @@ private: public: SubReferer(T* &t) : _t(t) {} ~SubReferer() { - if (_t != nullptr) _t->SubRef(); + if (_t != nullptr) _t->internal_subref(); } }; @@ -134,7 +134,7 @@ TEST("standalone") { } fprintf(stderr, "ping failed [retry %d]\n", retry); std::this_thread::sleep_for(200ms); - sb->SubRef(); + sb->internal_subref(); sb = orb.GetTarget(18541); } ASSERT_TRUE(checkOk(req)); diff --git a/slobrok/src/tests/startsome/rpc_info.cpp b/slobrok/src/tests/startsome/rpc_info.cpp index a37cfa24889..be6d59f6a81 100644 --- a/slobrok/src/tests/startsome/rpc_info.cpp +++ b/slobrok/src/tests/startsome/rpc_info.cpp @@ -13,16 +13,16 @@ public: void GetReq(FRT_RPCRequest **req, FRT_Supervisor *supervisor) { if ((*req) != nullptr) - (*req)->SubRef(); + (*req)->internal_subref(); (*req) = supervisor->AllocRPCRequest(); } void FreeReqs(FRT_RPCRequest *r1, FRT_RPCRequest *r2) { if (r1 != nullptr) - r1->SubRef(); + r1->internal_subref(); if (r2 != nullptr) - r2->SubRef(); + r2->internal_subref(); } void DumpMethodInfo(const char *indent, FRT_RPCRequest *info, @@ -123,7 +123,7 @@ public: m_list->GetErrorMessage()); } FreeReqs(m_list, info); - target->SubRef(); + target->internal_subref(); return 0; } }; diff --git a/slobrok/src/vespa/slobrok/sbmirror.cpp b/slobrok/src/vespa/slobrok/sbmirror.cpp index 62d288d40dc..3936b4dac4c 100644 --- a/slobrok/src/vespa/slobrok/sbmirror.cpp +++ b/slobrok/src/vespa/slobrok/sbmirror.cpp @@ -44,10 +44,10 @@ MirrorAPI::~MirrorAPI() _configurator.reset(0); if (_req != 0) { _req->Abort(); - _req->SubRef(); + _req->internal_subref(); } if (_target != 0) { - _target->SubRef(); + _target->internal_subref(); } } @@ -208,7 +208,7 @@ MirrorAPI::handleReconfig() std::string cps = _slobrokSpecs.logString(); LOG(warning, "current server %s not in list of location brokers: %s", _currSlobrok.c_str(), cps.c_str()); - _target->SubRef(); + _target->internal_subref(); _target = 0; } } @@ -224,7 +224,7 @@ MirrorAPI::handleReqDone() if (reconn) { if (_target != 0) { - _target->SubRef(); + _target->internal_subref(); } _target = 0; } else { diff --git a/slobrok/src/vespa/slobrok/sbregister.cpp b/slobrok/src/vespa/slobrok/sbregister.cpp index 925d4ea62bc..e7db255c5d6 100644 --- a/slobrok/src/vespa/slobrok/sbregister.cpp +++ b/slobrok/src/vespa/slobrok/sbregister.cpp @@ -90,10 +90,10 @@ RegisterAPI::~RegisterAPI() _configurator.reset(0); if (_req != 0) { _req->Abort(); - _req->SubRef(); + _req->internal_subref(); } if (_target != 0) { - _target->SubRef(); + _target->internal_subref(); } } @@ -139,7 +139,7 @@ RegisterAPI::handleReqDone() // unexpected error; close our connection to this // slobrok server and try again with a fresh slate if (_target != 0) { - _target->SubRef(); + _target->internal_subref(); } _target = 0; _busy.store(true, std::memory_order_relaxed); @@ -159,7 +159,7 @@ RegisterAPI::handleReqDone() // reset backoff strategy on any successful request _backOff.reset(); } - _req->SubRef(); + _req->internal_subref(); _req = 0; } } @@ -173,7 +173,7 @@ RegisterAPI::handleReconnect() vespalib::string cps = _slobrokSpecs.logString(); LOG(warning, "[RPC @ %s] location broker %s removed, will disconnect and use one of: %s", createSpec(_orb).c_str(), _currSlobrok.c_str(), cps.c_str()); - _target->SubRef(); + _target->internal_subref(); _target = 0; } } diff --git a/slobrok/src/vespa/slobrok/server/exchange_manager.cpp b/slobrok/src/vespa/slobrok/server/exchange_manager.cpp index 4b8c1c02252..94def0271c8 100644 --- a/slobrok/src/vespa/slobrok/server/exchange_manager.cpp +++ b/slobrok/src/vespa/slobrok/server/exchange_manager.cpp @@ -160,7 +160,7 @@ ExchangeManager::WorkPackage::WorkItem::RequestDone(FRT_RPCRequest *req) LOG(warning, "error doing workitem: %s", req->GetErrorMessage()); // XXX tell remslob? } - req->SubRef(); + req->internal_subref(); _pendingReq = nullptr; _pkg.doneItem(denied); } diff --git a/slobrok/src/vespa/slobrok/server/managed_rpc_server.cpp b/slobrok/src/vespa/slobrok/server/managed_rpc_server.cpp index 12761c0e6e9..a7013f6e04e 100644 --- a/slobrok/src/vespa/slobrok/server/managed_rpc_server.cpp +++ b/slobrok/src/vespa/slobrok/server/managed_rpc_server.cpp @@ -59,7 +59,7 @@ ManagedRpcServer::cleanupMonitor() { _monitor.disable(); if (_monitoredServer != nullptr) { - _monitoredServer->SubRef(); + _monitoredServer->internal_subref(); _monitoredServer = nullptr; } if (_checkServerReq != nullptr) { @@ -100,7 +100,7 @@ ManagedRpcServer::RequestDone(FRT_RPCRequest *req) if (req->GetErrorCode() == FRTE_RPC_ABORT) { LOG(debug, "rpcserver[%s].check aborted", getName().c_str()); - req->SubRef(); + req->internal_subref(); _checkServerReq = nullptr; return; } @@ -119,7 +119,7 @@ ManagedRpcServer::RequestDone(FRT_RPCRequest *req) } else { errmsg = "checkServer failed validation"; } - req->SubRef(); + req->internal_subref(); _checkServerReq = nullptr; cleanupMonitor(); _mmanager.notifyFailedRpcSrv(this, errmsg); @@ -130,7 +130,7 @@ ManagedRpcServer::RequestDone(FRT_RPCRequest *req) LOG_ASSERT(_monitoredServer != nullptr); _monitor.enable(_monitoredServer); - req->SubRef(); + req->internal_subref(); _checkServerReq = nullptr; _mmanager.notifyOkRpcSrv(this); } diff --git a/slobrok/src/vespa/slobrok/server/remote_slobrok.cpp b/slobrok/src/vespa/slobrok/server/remote_slobrok.cpp index aa891ae0048..0f6fffb7f4f 100644 --- a/slobrok/src/vespa/slobrok/server/remote_slobrok.cpp +++ b/slobrok/src/vespa/slobrok/server/remote_slobrok.cpp @@ -32,7 +32,7 @@ void RemoteSlobrok::shutdown() { _reconnecter.disable(); if (_remote != nullptr) { - _remote->SubRef(); + _remote->internal_subref(); _remote = nullptr; } @@ -110,7 +110,7 @@ void RemoteSlobrok::handleFetchResult() { _serviceMapMirror.clear(); success = false; } - _remFetchReq->SubRef(); + _remFetchReq->internal_subref(); _remFetchReq = nullptr; if (success) { maybeStartFetch(); @@ -132,12 +132,12 @@ RemoteSlobrok::RequestDone(FRT_RPCRequest *req) const char *myspec = args[1]._string._str; LOG(info, "addPeer(%s, %s) on remote slobrok %s at %s: %s", myname, myspec, getName().c_str(), getSpec().c_str(), req->GetErrorMessage()); - req->SubRef(); + req->internal_subref(); _remAddPeerReq = nullptr; fail(); return; } - req->SubRef(); + req->internal_subref(); _remAddPeerReq = nullptr; } else { LOG(error, "got unknown request back in RequestDone()"); @@ -166,7 +166,7 @@ RemoteSlobrok::fail() { // disconnect if (_remote != nullptr) { - _remote->SubRef(); + _remote->internal_subref(); _remote = nullptr; } // schedule reconnect attempt diff --git a/slobrok/src/vespa/slobrok/server/sbenv.h b/slobrok/src/vespa/slobrok/server/sbenv.h index 212c163d0cc..cdfe5a21667 100644 --- a/slobrok/src/vespa/slobrok/server/sbenv.h +++ b/slobrok/src/vespa/slobrok/server/sbenv.h @@ -15,7 +15,6 @@ #include <vespa/vespalib/net/http/simple_health_producer.h> #include <vespa/vespalib/net/http/simple_component_config_producer.h> -class FastOS_ThreadPool; class FNET_Transport; class FNET_Scheduler; class FRT_Supervisor; diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index 4b7c12b0f31..1062a89f055 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -2,7 +2,6 @@ vespa_define_module( DEPENDS vespadefaults - fastos metrics config_cloudconfig configdefinitions diff --git a/storage/src/tests/common/dummystoragelink.h b/storage/src/tests/common/dummystoragelink.h index e8ccc38df76..8da92917c08 100644 --- a/storage/src/tests/common/dummystoragelink.h +++ b/storage/src/tests/common/dummystoragelink.h @@ -11,8 +11,6 @@ #include <vespa/storage/common/bucketmessages.h> #include <vespa/storageapi/message/internal.h> -class FastOS_ThreadPool; - namespace storage { class DummyStorageLink : public StorageLink { diff --git a/storage/src/tests/common/metricstest.cpp b/storage/src/tests/common/metricstest.cpp index 78fa32e24e5..63d7be7c499 100644 --- a/storage/src/tests/common/metricstest.cpp +++ b/storage/src/tests/common/metricstest.cpp @@ -52,8 +52,7 @@ namespace { { framework::Clock& _clock; explicit MetricClock(framework::Clock& c) : _clock(c) {} - [[nodiscard]] time_t getTime() const override { return vespalib::count_s(_clock.getMonotonicTime().time_since_epoch()); } - [[nodiscard]] time_t getTimeInMilliSecs() const override { return vespalib::count_ms(_clock.getMonotonicTime().time_since_epoch()); } + [[nodiscard]] metrics::time_point getTime() const override { return _clock.getSystemTime(); } }; } @@ -85,10 +84,7 @@ void MetricsTest::SetUp() { _metricManager->registerMetric(guard, *_topSet); } - _metricsConsumer = std::make_unique<StatusMetricConsumer>( - _node->getComponentRegister(), - *_metricManager, - "status"); + _metricsConsumer = std::make_unique<StatusMetricConsumer>(_node->getComponentRegister(), *_metricManager, "status"); _filestorMetrics = std::make_shared<FileStorMetrics>(); _filestorMetrics->initDiskMetrics(1, 1); @@ -189,7 +185,7 @@ void MetricsTest::createFakeLoad() } _clock->addSecondsToTime(60); _metricManager->timeChangedNotification(); - while (int64_t(_metricManager->getLastProcessedTime()) < vespalib::count_s(_clock->getMonotonicTime().time_since_epoch())) { + while (_metricManager->getLastProcessedTime() < _clock->getSystemTime()) { std::this_thread::sleep_for(5ms); _metricManager->timeChangedNotification(); } @@ -239,7 +235,7 @@ TEST_F(MetricsTest, snapshot_presenting) { for (uint32_t i=0; i<6; ++i) { _clock->addSecondsToTime(60); _metricManager->timeChangedNotification(); - while (int64_t(_metricManager->getLastProcessedTime()) < vespalib::count_s(_clock->getMonotonicTime().time_since_epoch())) { + while (_metricManager->getLastProcessedTime() < _clock->getSystemTime()) { std::this_thread::sleep_for(1ms); } } @@ -299,8 +295,8 @@ MetricsTest::createSnapshotForPeriod(std::chrono::seconds secs) const { _clock->addSecondsToTime(secs.count()); _metricManager->timeChangedNotification(); - while (int64_t(_metricManager->getLastProcessedTime()) < vespalib::count_s(_clock->getMonotonicTime().time_since_epoch())) { - std::this_thread::sleep_for(100ms); + while (_metricManager->getLastProcessedTime() < _clock->getSystemTime()) { + std::this_thread::sleep_for(5ms); } } diff --git a/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp b/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp index bac1ab34574..ad410eb93e8 100644 --- a/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp +++ b/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp @@ -67,7 +67,7 @@ struct FixtureBase { // instance _before_ we destroy the request itself. dispatcher._enqueued.clear(); if (bound_request) { - bound_request->SubRef(); + bound_request->internal_subref(); } } }; diff --git a/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp b/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp index 9a98a40e7eb..bfc22b9f1ea 100644 --- a/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp +++ b/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp @@ -204,8 +204,8 @@ public: EXPECT_TRUE(req->IsError()); EXPECT_EQ(req->GetErrorCode(), FRTE_RPC_METHOD_FAILED); EXPECT_EQ(req->GetErrorMessage(), expected_msg); - target->SubRef(); - req->SubRef(); + target->internal_subref(); + req->internal_subref(); } }; diff --git a/storage/src/tests/storageserver/statereportertest.cpp b/storage/src/tests/storageserver/statereportertest.cpp index 1fb5a9730c4..380604ae77b 100644 --- a/storage/src/tests/storageserver/statereportertest.cpp +++ b/storage/src/tests/storageserver/statereportertest.cpp @@ -30,7 +30,6 @@ public: }; struct StateReporterTest : Test { - FastOS_ThreadPool _threadPool; framework::defaultimplementation::FakeClock* _clock; std::unique_ptr<TestServiceLayerApp> _node; std::unique_ptr<DummyStorageLink> _top; @@ -54,15 +53,13 @@ struct MetricClock : public metrics::MetricManager::Timer { framework::Clock& _clock; explicit MetricClock(framework::Clock& c) : _clock(c) {} - [[nodiscard]] time_t getTime() const override { return vespalib::count_s(_clock.getMonotonicTime().time_since_epoch()); } - [[nodiscard]] time_t getTimeInMilliSecs() const override { return vespalib::count_ms(_clock.getMonotonicTime().time_since_epoch()); } + [[nodiscard]] metrics::time_point getTime() const override { return _clock.getSystemTime(); } }; } StateReporterTest::StateReporterTest() - : _threadPool(), - _clock(nullptr), + : _clock(nullptr), _top(), _stateReporter() { @@ -87,11 +84,8 @@ void StateReporterTest::SetUp() { _metricManager->registerMetric(guard, *_topSet); } - _stateReporter = std::make_unique<StateReporter>( - _node->getComponentRegister(), - *_metricManager, - _generationFetcher, - "status"); + _stateReporter = std::make_unique<StateReporter>(_node->getComponentRegister(), *_metricManager, + _generationFetcher, "status"); _filestorMetrics = std::make_shared<FileStorMetrics>(); _filestorMetrics->initDiskMetrics(1, 1); @@ -127,20 +121,14 @@ vespalib::Slime slime; \ #define ASSERT_GENERATION(jsonData, component, generation) \ { \ PARSE_JSON(jsonData); \ - ASSERT_EQ( \ - generation, \ - slime.get()["config"][component]["generation"].asDouble()); \ + ASSERT_EQ(generation, slime.get()["config"][component]["generation"].asDouble()); \ } #define ASSERT_NODE_STATUS(jsonData, code, message) \ { \ PARSE_JSON(jsonData); \ - ASSERT_EQ( \ - vespalib::string(code), \ - slime.get()["status"]["code"].asString().make_string()); \ - ASSERT_EQ( \ - vespalib::string(message), \ - slime.get()["status"]["message"].asString().make_string()); \ + ASSERT_EQ(vespalib::string(code), slime.get()["status"]["code"].asString().make_string()); \ + ASSERT_EQ(vespalib::string(message), slime.get()["status"]["message"].asString().make_string()); \ } #define ASSERT_METRIC_GET_PUT(jsonData, expGetCount, expPutCount) \ @@ -150,16 +138,11 @@ vespalib::Slime slime; \ double putCount = -1; \ size_t metricCount = slime.get()["metrics"]["values"].children(); \ for (size_t j=0; j<metricCount; j++) { \ - const vespalib::string name = slime.get()["metrics"]["values"][j]["name"] \ - .asString().make_string(); \ - if (name.compare("vds.filestor.allthreads.get.count") == 0) \ - { \ - getCount = slime.get()["metrics"]["values"][j]["values"]["count"] \ - .asDouble(); \ - } else if (name.compare("vds.filestor.allthreads.put.count") == 0) \ - { \ - putCount = slime.get()["metrics"]["values"][j]["values"]["count"] \ - .asDouble(); \ + const vespalib::string name = slime.get()["metrics"]["values"][j]["name"].asString().make_string(); \ + if (name.compare("vds.filestor.allthreads.get.count") == 0) { \ + getCount = slime.get()["metrics"]["values"][j]["values"]["count"].asDouble(); \ + } else if (name.compare("vds.filestor.allthreads.put.count") == 0) { \ + putCount = slime.get()["metrics"]["values"][j]["values"]["count"].asDouble(); \ } \ } \ ASSERT_EQ(expGetCount, getCount); \ @@ -228,8 +211,7 @@ TEST_F(StateReporterTest, report_metrics) { for (uint32_t i = 0; i < 6; ++i) { _clock->addSecondsToTime(60); _metricManager->timeChangedNotification(); - while (int64_t(_metricManager->getLastProcessedTime()) < vespalib::count_s(_clock->getMonotonicTime().time_since_epoch())) - { + while (_metricManager->getLastProcessedTime() < _clock->getSystemTime()) { std::this_thread::sleep_for(1ms); } } diff --git a/storage/src/vespa/storage/common/statusmetricconsumer.cpp b/storage/src/vespa/storage/common/statusmetricconsumer.cpp index 8eb3e9f3ab6..68866027cd1 100644 --- a/storage/src/vespa/storage/common/statusmetricconsumer.cpp +++ b/storage/src/vespa/storage/common/statusmetricconsumer.cpp @@ -5,7 +5,6 @@ #include <boost/lexical_cast.hpp> #include <vespa/metrics/jsonwriter.h> #include <vespa/metrics/textwriter.h> -#include <vespa/metrics/xmlwriter.h> #include <vespa/metrics/metricmanager.h> #include <vespa/storageapi/messageapi/storagemessage.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -37,10 +36,6 @@ StatusMetricConsumer::getReportContentType(const framework::HttpUrlPath& path) c return "text/plain"; } - if (path.getAttribute("format") == "xml") { - return "application/xml"; - } - if (path.getAttribute("format") == "text") { return "text/plain"; } @@ -56,18 +51,9 @@ bool StatusMetricConsumer::reportStatus(std::ostream& out, const framework::HttpUrlPath& path) const { - // Update metrics unless 'dontcallupdatehooks' is 1. Update - // snapshot metrics too, if callsnapshothooks is set to 1. - if (path.get("dontcallupdatehooks", 0) == 0) { - bool updateSnapshotHooks = path.get("callsnapshothooks", 0) == 1; - LOG(debug, "Updating metrics ahead of status page view%s", - updateSnapshotHooks ? ", calling snapshot hooks too" : "."); - _manager.updateMetrics(updateSnapshotHooks); - } else { - LOG(debug, "Not calling update hooks as dontcallupdatehooks option has been given"); - } - int64_t currentTimeS(vespalib::count_s(_component.getClock().getMonotonicTime().time_since_epoch())); - bool xml = (path.getAttribute("format") == "xml"); + _manager.updateMetrics(); + + vespalib::system_time currentTime = _component.getClock().getSystemTime(); bool json = (path.getAttribute("format") == "json"); int verbosity(path.get("verbosity", 0)); @@ -78,52 +64,53 @@ StatusMetricConsumer::reportStatus(std::ostream& out, if (path.hasAttribute("task") && path.getAttribute("task") == "reset") { std::lock_guard guard(_lock); - _manager.reset(currentTimeS); + _manager.reset(currentTime); } if (path.hasAttribute("interval")) { // Grab the snapshot we want to view more of - int32_t interval(boost::lexical_cast<int32_t>(path.getAttribute("interval"))); + int32_t intervalS(boost::lexical_cast<int32_t>(path.getAttribute("interval"))); metrics::MetricLockGuard metricLock(_manager.getMetricLock()); std::unique_ptr<metrics::MetricSnapshot> generated; const metrics::MetricSnapshot* snapshot; - if (interval == -2) { + if (intervalS == -2) { snapshot = &_manager.getActiveMetrics(metricLock); - _manager.getActiveMetrics(metricLock).setToTime(currentTimeS); - } else if (interval == -1) { + _manager.getActiveMetrics(metricLock).setToTime(currentTime); + } else if (intervalS == -1) { // "Prime" the metric structure by first fetching the set of active // metrics (complete with structure) and resetting these. This // leaves us with an empty metrics set to which we can (in order) // add the total and the active metrics. If this is not done, non- // written metrics won't be included even if copyUnset is true. generated = std::make_unique<metrics::MetricSnapshot>( - "Total metrics from start until current time", 0, + "Total metrics from start until current time", 0s, _manager.getActiveMetrics(metricLock).getMetrics(), copyUnset); - generated->reset(0); - _manager.getTotalMetricSnapshot(metricLock).addToSnapshot(*generated, currentTimeS); - _manager.getActiveMetrics(metricLock).addToSnapshot(*generated, currentTimeS); + generated->reset(); + _manager.getTotalMetricSnapshot(metricLock).addToSnapshot(*generated, currentTime); + _manager.getActiveMetrics(metricLock).addToSnapshot(*generated, currentTime); generated->setFromTime(_manager.getTotalMetricSnapshot(metricLock).getFromTime()); snapshot = generated.get(); - } else if (interval == 0) { + } else if (intervalS == 0) { if (copyUnset) { generated = std::make_unique<metrics::MetricSnapshot>( - _manager.getTotalMetricSnapshot(metricLock).getName(), 0, + _manager.getTotalMetricSnapshot(metricLock).getName(), 0s, _manager.getActiveMetrics(metricLock).getMetrics(), true); - generated->reset(0); - _manager.getTotalMetricSnapshot(metricLock).addToSnapshot(*generated, currentTimeS); + generated->reset(); + _manager.getTotalMetricSnapshot(metricLock).addToSnapshot(*generated, currentTime); snapshot = generated.get(); } else { snapshot = &_manager.getTotalMetricSnapshot(metricLock); } } else { + vespalib::duration interval = vespalib::from_s(intervalS); if (copyUnset) { generated = std::make_unique<metrics::MetricSnapshot>( - _manager.getMetricSnapshot(metricLock, interval).getName(), 0, + _manager.getMetricSnapshot(metricLock, interval).getName(), 0s, _manager.getActiveMetrics(metricLock).getMetrics(), true); - generated->reset(0); + generated->reset(); _manager.getMetricSnapshot(metricLock, interval, temporarySnap) - .addToSnapshot(*generated, currentTimeS); + .addToSnapshot(*generated, currentTime); snapshot = generated.get(); } else { snapshot = &_manager.getMetricSnapshot(metricLock, interval, temporarySnap); @@ -131,13 +118,7 @@ StatusMetricConsumer::reportStatus(std::ostream& out, } std::string consumer = path.getAttribute("consumer", ""); - if (xml) { - out << "<?xml version=\"1.0\"?>\n"; - vespalib::XmlOutputStream xos(out); - metrics::XmlWriter xmlWriter(xos, snapshot->getPeriod(), verbosity); - _manager.visit(metricLock, *snapshot, xmlWriter, consumer); - out << "\n"; - } else if (json) { + if (json) { vespalib::asciistream jsonStreamData; vespalib::JsonStream stream(jsonStreamData, true); stream << Object() << "metrics"; diff --git a/storage/src/vespa/storage/distributor/distributor_stripe_pool.cpp b/storage/src/vespa/storage/distributor/distributor_stripe_pool.cpp index 26ca8963783..ceadd20baca 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe_pool.cpp +++ b/storage/src/vespa/storage/distributor/distributor_stripe_pool.cpp @@ -8,8 +8,7 @@ namespace storage::distributor { DistributorStripePool::DistributorStripePool(bool test_mode, PrivateCtorTag) - : _thread_pool(std::make_unique<FastOS_ThreadPool>()), - _n_stripe_bits(0), + : _n_stripe_bits(0), _stripes(), _threads(), _mutex(), @@ -119,7 +118,7 @@ void DistributorStripePool::start(const std::vector<TickableStripe*>& stripes) { } std::unique_lock lock(_mutex); // Ensure _threads is visible to all started threads for (auto& s : _stripes) { - _threads.emplace_back(_thread_pool->NewThread(s.get())); + _threads.start([ptr = s.get()](){ ptr->run(); }); } } @@ -131,9 +130,7 @@ void DistributorStripePool::stop_and_join() { for (auto& s : _stripes) { s->signal_should_stop(); } - for (auto* t : _threads) { - t->Join(); - } + _threads.join(); } void DistributorStripePool::set_tick_wait_duration(vespalib::duration new_tick_wait_duration) noexcept { diff --git a/storage/src/vespa/storage/distributor/distributor_stripe_pool.h b/storage/src/vespa/storage/distributor/distributor_stripe_pool.h index 00f5f57edf9..6ac95c27b76 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe_pool.h +++ b/storage/src/vespa/storage/distributor/distributor_stripe_pool.h @@ -2,14 +2,12 @@ #pragma once #include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/util/thread.h> #include <atomic> #include <condition_variable> #include <mutex> #include <vector> -class FastOS_ThreadInterface; -class FastOS_ThreadPool; - namespace storage::distributor { class DistributorStripeThread; @@ -37,12 +35,10 @@ class TickableStripe; */ class DistributorStripePool { using StripeVector = std::vector<std::unique_ptr<DistributorStripeThread>>; - using NativeThreadVector = std::vector<FastOS_ThreadInterface*>; - std::unique_ptr<FastOS_ThreadPool> _thread_pool; uint8_t _n_stripe_bits; StripeVector _stripes; - NativeThreadVector _threads; + vespalib::ThreadPool _threads; std::mutex _mutex; std::condition_variable _parker_cond; size_t _parked_threads; // Must be protected by _park_mutex diff --git a/storage/src/vespa/storage/distributor/distributor_stripe_thread.cpp b/storage/src/vespa/storage/distributor/distributor_stripe_thread.cpp index 8f37dbbbf5d..72854d9af75 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe_thread.cpp +++ b/storage/src/vespa/storage/distributor/distributor_stripe_thread.cpp @@ -23,7 +23,7 @@ DistributorStripeThread::DistributorStripeThread(TickableStripe& stripe, DistributorStripeThread::~DistributorStripeThread() = default; -void DistributorStripeThread::Run(FastOS_ThreadInterface*, void*) { +void DistributorStripeThread::run() { uint32_t tick_waits_inhibited = 0; while (!should_stop_thread_relaxed()) { while (should_park_relaxed()) { diff --git a/storage/src/vespa/storage/distributor/distributor_stripe_thread.h b/storage/src/vespa/storage/distributor/distributor_stripe_thread.h index 7015d27a53e..8b9453ab3f3 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe_thread.h +++ b/storage/src/vespa/storage/distributor/distributor_stripe_thread.h @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fastos/thread.h> #include <vespa/vespalib/util/time.h> #include <atomic> #include <condition_variable> @@ -21,7 +20,7 @@ class TickableStripe; * A DistributorStripeThread instance is bidirectionally bound to a particular pool and * should therefore always be created by the pool itself (never standalone). */ -class DistributorStripeThread : private FastOS_Runnable { +class DistributorStripeThread { using AtomicDuration = std::atomic<vespalib::duration>; TickableStripe& _stripe; @@ -41,7 +40,7 @@ public: DistributorStripePool& stripe_pool); ~DistributorStripeThread(); - void Run(FastOS_ThreadInterface*, void*) override; + void run(); // Wakes up stripe thread if it's currently waiting for an external event to be triggered, // such as the arrival of a new RPC message. If thread is parked this call will have no diff --git a/storage/src/vespa/storage/distributor/messagetracker.cpp b/storage/src/vespa/storage/distributor/messagetracker.cpp index 93db31bdc29..8830e5ecabc 100644 --- a/storage/src/vespa/storage/distributor/messagetracker.cpp +++ b/storage/src/vespa/storage/distributor/messagetracker.cpp @@ -3,6 +3,7 @@ #include "messagetracker.h" #include <vespa/storageapi/messageapi/bucketcommand.h> #include <vespa/storageapi/messageapi/bucketreply.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".messagetracker"); diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp index 55fe2e039e1..bdf4fa2ba72 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp @@ -12,7 +12,7 @@ #include <vespa/storage/distributor/distributor_bucket_space_repo.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/vdslib/state/cluster_state_bundle.h> -#include <vespa/vespalib/stllike/hash_map.hpp> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".distributor.callback.twophaseupdate"); diff --git a/storage/src/vespa/storage/distributor/operations/external/visitoroperation.cpp b/storage/src/vespa/storage/distributor/operations/external/visitoroperation.cpp index 4d40e93477f..9e9196dbee7 100644 --- a/storage/src/vespa/storage/distributor/operations/external/visitoroperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/visitoroperation.cpp @@ -12,6 +12,7 @@ #include <vespa/document/base/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> #include <sstream> +#include <optional> #include <vespa/log/log.h> LOG_SETUP(".visitoroperation"); @@ -88,7 +89,7 @@ VisitorOperation::VisitorOperation( { const std::vector<document::BucketId>& buckets = m->getBuckets(); - if (buckets.size() > 0) { + if (!buckets.empty()) { _superBucket = SuperBucketInfo(buckets[0]); } @@ -114,12 +115,10 @@ VisitorOperation::getLastBucketVisited() LOG(spam, "getLastBucketVisited(): Sub bucket count: %zu", _superBucket.subBucketsVisitOrder.size()); - for (uint32_t i=0; i<_superBucket.subBucketsVisitOrder.size(); i++) { - auto found = _superBucket.subBuckets.find(_superBucket.subBucketsVisitOrder[i]); + for (const auto& sub_bucket : _superBucket.subBucketsVisitOrder) { + auto found = _superBucket.subBuckets.find(sub_bucket); assert(found != _superBucket.subBuckets.end()); - LOG(spam, "%s => %s", - found->first.toString().c_str(), - found->second.toString().c_str()); + LOG(spam, "%s => %s", found->first.toString().c_str(), found->second.toString().c_str()); if (found->second.done) { foundDone = true; @@ -151,9 +150,7 @@ VisitorOperation::timeLeft() const noexcept { const auto elapsed = _operationTimer.getElapsedTime(); - - LOG(spam, - "Checking if visitor has timed out: elapsed=%" PRId64 " ms, timeout=%" PRId64 " ms", + LOG(spam, "Checking if visitor has timed out: elapsed=%" PRId64 " ms, timeout=%" PRId64 " ms", vespalib::count_ms(elapsed), vespalib::count_ms(_msg->getTimeout())); @@ -168,7 +165,7 @@ void VisitorOperation::markCompleted(const document::BucketId& bid, const api::ReturnCode& code) { - VisitBucketMap::iterator found = _superBucket.subBuckets.find(bid); + auto found = _superBucket.subBuckets.find(bid); assert(found != _superBucket.subBuckets.end()); BucketInfo& info = found->second; @@ -196,11 +193,11 @@ VisitorOperation::onReceive( DistributorStripeMessageSender& sender, const api::StorageReply::SP& r) { - api::CreateVisitorReply& reply = static_cast<api::CreateVisitorReply&>(*r); + auto& reply = dynamic_cast<api::CreateVisitorReply&>(*r); _trace.add(reply.steal_trace()); - SentMessagesMap::iterator iter = _sentMessages.find(reply.getMsgId()); + auto iter = _sentMessages.find(reply.getMsgId()); assert(iter != _sentMessages.end()); api::CreateVisitorCommand& storageVisitor = *iter->second; @@ -223,9 +220,8 @@ VisitorOperation::onReceive( } // else: will lose code for non-critical events, degenerates to "not found". - for (uint32_t i = 0; i < storageVisitor.getBuckets().size(); i++) { - const document::BucketId& bid(storageVisitor.getBuckets()[i]); - markCompleted(bid, result); + for (const auto& bucket : storageVisitor.getBuckets()) { + markCompleted(bucket, result); } _sentMessages.erase(iter); @@ -234,15 +230,14 @@ VisitorOperation::onReceive( namespace { -class VisitorVerificationException -{ +class VisitorVerificationException { public: VisitorVerificationException(api::ReturnCode::Result result, vespalib::stringref message) : _code(result, message) {} - const api::ReturnCode& getReturnCode() const { + const api::ReturnCode& getReturnCode() const noexcept { return _code; } @@ -438,10 +433,11 @@ namespace { struct NextEntryFinder : public BucketDatabase::EntryProcessor { bool _first; document::BucketId _last; - std::unique_ptr<document::BucketId> _next; + std::optional<document::BucketId> _next; - NextEntryFinder(const document::BucketId& id) - : _first(true), _last(id), _next() {} + explicit NextEntryFinder(const document::BucketId& id) noexcept + : _first(true), _last(id), _next() + {} bool process(const BucketDatabase::ConstEntryRef& e) override { document::BucketId bucket(e.getBucketId()); @@ -450,27 +446,26 @@ struct NextEntryFinder : public BucketDatabase::EntryProcessor { _first = false; return true; } else { - _next.reset(new document::BucketId(bucket)); + _next.emplace(bucket); return false; } } }; -std::unique_ptr<document::BucketId> -getBucketIdAndLast( - BucketDatabase& database, - const document::BucketId& super, - const document::BucketId& last) +std::optional<document::BucketId> +getBucketIdAndLast(BucketDatabase& database, + const document::BucketId& super, + const document::BucketId& last) { if (!super.contains(last)) { NextEntryFinder proc(super); database.forEach(proc, super); - return std::move(proc._next); + return proc._next; } else { NextEntryFinder proc(last); database.forEach(proc, last); - return std::move(proc._next); + return proc._next; } } @@ -481,12 +476,12 @@ VisitorOperation::expandBucketContained() { uint32_t maxBuckets = _msg->getMaxBucketsPerVisitor(); - std::unique_ptr<document::BucketId> bid = getBucketIdAndLast( + std::optional<document::BucketId> bid = getBucketIdAndLast( _bucketSpace.getBucketDatabase(), _superBucket.bid, _lastBucket); - while (bid.get() && _superBucket.subBuckets.size() < maxBuckets) { + while (bid.has_value() && _superBucket.subBuckets.size() < maxBuckets) { if (!_superBucket.bid.contains(*bid)) { LOG(spam, "Iterating: Found bucket %s is not contained in bucket %s", @@ -502,7 +497,7 @@ VisitorOperation::expandBucketContained() bid = getBucketIdAndLast(_bucketSpace.getBucketDatabase(), _superBucket.bid, *bid); } - bool doneExpand = (!bid.get() || !_superBucket.bid.contains(*bid)); + bool doneExpand = (!bid.has_value() || !_superBucket.bid.contains(*bid)); return doneExpand; } @@ -541,15 +536,8 @@ VisitorOperation::expandBucket() namespace { -bool -alreadyTried(const std::vector<uint16_t>& triedNodes, uint16_t node) -{ - for (uint32_t j = 0; j < triedNodes.size(); j++) { - if (triedNodes[j] == node) { - return true; - } - } - return false; +[[nodiscard]] bool alreadyTried(const std::vector<uint16_t>& triedNodes, uint16_t node) noexcept { + return std::find(triedNodes.begin(), triedNodes.end(), node) != triedNodes.end(); } int diff --git a/storage/src/vespa/storage/distributor/sentmessagemap.cpp b/storage/src/vespa/storage/distributor/sentmessagemap.cpp index 44dd4fbde89..4b7292c1e81 100644 --- a/storage/src/vespa/storage/distributor/sentmessagemap.cpp +++ b/storage/src/vespa/storage/distributor/sentmessagemap.cpp @@ -4,6 +4,7 @@ #include <vespa/storage/distributor/operations/operation.h> #include <sstream> #include <set> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".distributor.callback.map"); diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt index 8b82bb251b2..009f8170669 100644 --- a/storage/src/vespa/storage/storageserver/CMakeLists.txt +++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt @@ -14,7 +14,6 @@ vespa_add_library(storage_storageserver OBJECT documentapiconverter.cpp fnet_metrics_wrapper.cpp mergethrottler.cpp - messagesink.cpp opslogger.cpp priorityconverter.cpp rpcrequestwrapper.cpp diff --git a/storage/src/vespa/storage/storageserver/messagesink.cpp b/storage/src/vespa/storage/storageserver/messagesink.cpp deleted file mode 100644 index 94762e545d9..00000000000 --- a/storage/src/vespa/storage/storageserver/messagesink.cpp +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "messagesink.h" -#include <vespa/storageapi/message/persistence.h> -#include <ostream> - -using std::shared_ptr; - -namespace storage { - -MessageSink::MessageSink() - : StorageLink("Message Sink") -{ -} - -MessageSink::~MessageSink() -{ - closeNextLink(); -} - -void -MessageSink::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - out << "MessageSink"; -} - -namespace { -#if 0 - std::string getTimeString() { - char timeBuf[200]; - time_t tm; - struct tm tms; - time(&tm); - gmtime_r(&tm, &tms); - strftime(timeBuf, sizeof(timeBuf), "%Y-%m-%d:%H:%M:%S %Z", &tms); - return std::string(timeBuf); - } -#endif -} - -IMPL_MSG_COMMAND_H(MessageSink, Get) -{ - //LOG(event, "[%s] Get %s", getTimeString().c_str(), - // cmd->getDocumentId()->toString()); - shared_ptr<api::StorageReply> rmsg(new api::GetReply(*cmd)); - rmsg->setResult(api::ReturnCode::NOT_IMPLEMENTED); - sendUp(rmsg); - return true; -} - -IMPL_MSG_COMMAND_H(MessageSink, Put) -{ - //LOG(event, "[%s] Put %s", getTimeString().c_str(), - // cmd->getDocumentId()->toString()); - shared_ptr<api::StorageReply> rmsg(new api::PutReply(*cmd)); - rmsg->setResult(api::ReturnCode::OK); - sendUp(rmsg); - return true; -} - -IMPL_MSG_COMMAND_H(MessageSink, Remove) -{ - //LOG(event, "[%s] Remove %s", getTimeString().c_str(), - // cmd->getDocumentId()->toString()); - shared_ptr<api::StorageReply> rmsg(new api::RemoveReply(*cmd)); - rmsg->setResult(api::ReturnCode::OK); - sendUp(rmsg); - return true; -} - -IMPL_MSG_COMMAND_H(MessageSink, Revert) -{ - //LOG(event, "[%s] Revert %s", getTimeString().c_str(), - // cmd->getDocumentId()->toString()); - shared_ptr<api::StorageReply> rmsg(new api::RevertReply(*cmd)); - rmsg->setResult(api::ReturnCode::OK); - sendUp(rmsg); - return true; -} - -} // storage diff --git a/storage/src/vespa/storage/storageserver/messagesink.h b/storage/src/vespa/storage/storageserver/messagesink.h deleted file mode 100644 index d98d0439b48..00000000000 --- a/storage/src/vespa/storage/storageserver/messagesink.h +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class storage::MessageSink - * @ingroup storageserver - * - * @brief This class grabs persistence messages, and answers them without doing anything. - * - * @version $Id$ - */ - -#pragma once - -#include <vespa/storage/common/storagelink.h> - -namespace storage { - -class MessageSink : public StorageLink { -public: - explicit MessageSink(); - MessageSink(const MessageSink &) = delete; - MessageSink& operator=(const MessageSink &) = delete; - ~MessageSink(); - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - -private: - DEF_MSG_COMMAND_H(Get); - DEF_MSG_COMMAND_H(Put); - DEF_MSG_COMMAND_H(Remove); - DEF_MSG_COMMAND_H(Revert); -}; - -} diff --git a/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp index e940dc71722..5e4cb9d3026 100644 --- a/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "rpc_target.h" #include "shared_rpc_resources.h" -#include <vespa/fastos/thread.h> #include <vespa/fnet/frt/supervisor.h> #include <vespa/fnet/frt/target.h> #include <vespa/fnet/transport.h> @@ -36,7 +35,7 @@ public: _spec(spec) {} ~RpcTargetImpl() override { - _target->SubRef(); + _target->internal_subref(); } FRT_Target* get() noexcept override { return _target; } bool is_valid() const noexcept override { return _target->IsValid(); } @@ -66,8 +65,7 @@ SharedRpcResources::SharedRpcResources(const config::ConfigUri& config_uri, int rpc_server_port, size_t rpc_thread_pool_size, size_t rpc_events_before_wakeup) - : _thread_pool(std::make_unique<FastOS_ThreadPool>()), - _transport(std::make_unique<FNET_Transport>(fnet::TransportConfig(rpc_thread_pool_size). + : _transport(std::make_unique<FNET_Transport>(fnet::TransportConfig(rpc_thread_pool_size). events_before_wakeup(rpc_events_before_wakeup))), _orb(std::make_unique<FRT_Supervisor>(_transport.get())), _slobrok_register(std::make_unique<slobrok::api::RegisterAPI>(*_orb, slobrok::ConfiguratorFactory(config_uri))), @@ -92,7 +90,7 @@ void SharedRpcResources::start_server_and_register_slobrok(vespalib::stringref m if (!_orb->Listen(_rpc_server_port)) { throw IllegalStateException(fmt("Failed to listen to RPC port %d", _rpc_server_port), VESPA_STRLOC); } - _transport->Start(_thread_pool.get()); + _transport->Start(); _slobrok_register->registerName(my_handle); wait_until_slobrok_is_ready(); _handle = my_handle; diff --git a/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.h b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.h index a30fcdc4ea7..953492089c1 100644 --- a/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.h +++ b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.h @@ -6,7 +6,6 @@ #include <vespa/vespalib/stllike/string.h> #include <memory> -class FastOS_ThreadPool; class FNET_Transport; class FRT_Supervisor; @@ -19,7 +18,6 @@ namespace storage::rpc { class SharedRpcResources { class RpcTargetFactoryImpl; - std::unique_ptr<FastOS_ThreadPool> _thread_pool; std::unique_ptr<FNET_Transport> _transport; std::unique_ptr<FRT_Supervisor> _orb; std::unique_ptr<slobrok::api::RegisterAPI> _slobrok_register; diff --git a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp index bcb5dbab279..e494f4e67da 100644 --- a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp @@ -83,7 +83,7 @@ namespace { struct SubRefDeleter { template <typename T> void operator()(T* v) const noexcept { - v->SubRef(); + v->internal_subref(); } }; diff --git a/storage/src/vespa/storage/storageserver/statemanager.cpp b/storage/src/vespa/storage/storageserver/statemanager.cpp index c59f7797bb8..654fe0e1f5d 100644 --- a/storage/src/vespa/storage/storageserver/statemanager.cpp +++ b/storage/src/vespa/storage/storageserver/statemanager.cpp @@ -18,7 +18,6 @@ #include <vespa/vespalib/util/string_escape.h> #include <vespa/vespalib/util/stringfmt.h> #include <fstream> -#include <ranges> #include <vespa/log/log.h> LOG_SETUP(".state.manager"); @@ -545,10 +544,9 @@ StateManager::getNodeInfo() const stream << "metrics"; try { metrics::MetricLockGuard lock(_metricManager.getMetricLock()); - std::vector<uint32_t> periods(_metricManager.getSnapshotPeriods(lock)); + auto periods(_metricManager.getSnapshotPeriods(lock)); if (!periods.empty()) { - uint32_t period = periods[0]; - const metrics::MetricSnapshot& snapshot(_metricManager.getMetricSnapshot(lock, period)); + const metrics::MetricSnapshot& snapshot(_metricManager.getMetricSnapshot(lock, periods[0])); metrics::JsonWriter metricJsonWriter(stream); _metricManager.visit(lock, snapshot, metricJsonWriter, "fleetcontroller"); } else { diff --git a/storage/src/vespa/storage/storageserver/statereporter.cpp b/storage/src/vespa/storage/storageserver/statereporter.cpp index 205a2d710a6..16de56fad22 100644 --- a/storage/src/vespa/storage/storageserver/statereporter.cpp +++ b/storage/src/vespa/storage/storageserver/statereporter.cpp @@ -69,11 +69,11 @@ vespalib::string StateReporter::getMetrics(const vespalib::string &consumer) { metrics::MetricLockGuard guard(_manager.getMetricLock()); - std::vector<uint32_t> periods = _manager.getSnapshotPeriods(guard); + auto periods = _manager.getSnapshotPeriods(guard); if (periods.empty()) { return ""; // no configuration yet } - uint32_t interval = periods[0]; + auto interval = periods[0]; // To get unset metrics, we have to copy active metrics, clear them // and then assign the snapshot @@ -81,9 +81,8 @@ StateReporter::getMetrics(const vespalib::string &consumer) _manager.getMetricSnapshot(guard, interval).getName(), interval, _manager.getActiveMetrics(guard).getMetrics(), true); - snapshot.reset(0); - _manager.getMetricSnapshot(guard, interval).addToSnapshot( - snapshot, vespalib::count_s(_component.getClock().getSystemTime().time_since_epoch())); + snapshot.reset(); + _manager.getMetricSnapshot(guard, interval).addToSnapshot(snapshot, _component.getClock().getSystemTime()); vespalib::asciistream json; vespalib::JsonStream stream(json); diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index a09abb25f7a..9f8456afc37 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -33,34 +33,36 @@ namespace storage { namespace { - using vespalib::getLastErrorString; +using vespalib::getLastErrorString; - void writePidFile(const vespalib::string& pidfile) - { - ssize_t rv = -1; - vespalib::string mypid = vespalib::make_string("%d\n", getpid()); - size_t lastSlash = pidfile.rfind('/'); - if (lastSlash != vespalib::string::npos) { - std::filesystem::create_directories(std::filesystem::path(pidfile.substr(0, lastSlash))); - } - int fd = open(pidfile.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); - if (fd != -1) { - rv = write(fd, mypid.c_str(), mypid.size()); - close(fd); - } - if (rv < 1) { - LOG(warning, "Failed to write pidfile '%s': %s", - pidfile.c_str(), getLastErrorString().c_str()); - } +void +writePidFile(const vespalib::string& pidfile) +{ + ssize_t rv = -1; + vespalib::string mypid = vespalib::make_string("%d\n", getpid()); + size_t lastSlash = pidfile.rfind('/'); + if (lastSlash != vespalib::string::npos) { + std::filesystem::create_directories(std::filesystem::path(pidfile.substr(0, lastSlash))); + } + int fd = open(pidfile.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd != -1) { + rv = write(fd, mypid.c_str(), mypid.size()); + close(fd); } + if (rv < 1) { + LOG(warning, "Failed to write pidfile '%s': %s", + pidfile.c_str(), getLastErrorString().c_str()); + } +} - void removePidFile(const vespalib::string& pidfile) - { - if (unlink(pidfile.c_str()) != 0) { - LOG(warning, "Failed to delete pidfile '%s': %s", - pidfile.c_str(), getLastErrorString().c_str()); - } +void +removePidFile(const vespalib::string& pidfile) +{ + if (unlink(pidfile.c_str()) != 0) { + LOG(warning, "Failed to delete pidfile '%s': %s", + pidfile.c_str(), getLastErrorString().c_str()); } +} } // End of anonymous namespace @@ -429,7 +431,8 @@ StorageNode::shutdown() LOG(debug, "Done shutting down node"); } -void StorageNode::configure(std::unique_ptr<StorServerConfig> config) { +void +StorageNode::configure(std::unique_ptr<StorServerConfig> config) { log_config_received(*config); // When we get config, we try to grab the config lock to ensure noone // else is doing configuration work, and then we write the new config @@ -445,7 +448,8 @@ void StorageNode::configure(std::unique_ptr<StorServerConfig> config) { } } -void StorageNode::configure(std::unique_ptr<UpgradingConfig> config) { +void +StorageNode::configure(std::unique_ptr<UpgradingConfig> config) { log_config_received(*config); { std::lock_guard configLockGuard(_configLock); @@ -457,7 +461,8 @@ void StorageNode::configure(std::unique_ptr<UpgradingConfig> config) { } } -void StorageNode::configure(std::unique_ptr<StorDistributionConfig> config) { +void +StorageNode::configure(std::unique_ptr<StorDistributionConfig> config) { log_config_received(*config); { std::lock_guard configLockGuard(_configLock); @@ -486,7 +491,8 @@ StorageNode::configure(std::unique_ptr<document::config::DocumenttypesConfig> co } } -void StorageNode::configure(std::unique_ptr<BucketspacesConfig> config) { +void +StorageNode::configure(std::unique_ptr<BucketspacesConfig> config) { log_config_received(*config); { std::lock_guard configLockGuard(_configLock); diff --git a/storage/src/vespa/storage/tools/storage-cmd.cpp b/storage/src/vespa/storage/tools/storage-cmd.cpp index 12299c7458e..bc932fcf6fd 100644 --- a/storage/src/vespa/storage/tools/storage-cmd.cpp +++ b/storage/src/vespa/storage/tools/storage-cmd.cpp @@ -90,7 +90,7 @@ public: req->GetErrorMessage()); continue; } - req->SubRef(); + req->internal_subref(); } FRT_RPCRequest *req = supervisor.supervisor().AllocRPCRequest(); @@ -115,8 +115,8 @@ public: req->GetErrorMessage()); } } - req->SubRef(); - target->SubRef(); + req->internal_subref(); + target->internal_subref(); } return retCode; } diff --git a/storage/src/vespa/storageapi/mbusprot/serializationhelper.h b/storage/src/vespa/storageapi/mbusprot/serializationhelper.h index 457a6178704..671ffbddd6f 100644 --- a/storage/src/vespa/storageapi/mbusprot/serializationhelper.h +++ b/storage/src/vespa/storageapi/mbusprot/serializationhelper.h @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fastos/types.h> #include <vespa/document/base/globalid.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/util/bytebuffer.h> @@ -13,12 +12,6 @@ namespace storage::mbusprot { class SerializationHelper { public: - static int64_t getLong(document::ByteBuffer& buf) { - int64_t tmp; - buf.getLongNetwork(tmp); - return tmp; - } - static int32_t getInt(document::ByteBuffer& buf) { int32_t tmp; buf.getIntNetwork(tmp); @@ -46,26 +39,6 @@ public: return s; } - static bool getBoolean(document::ByteBuffer& buf) { - uint8_t tmp; - buf.getByte(tmp); - return (tmp == 1); - } - - static api::ReturnCode getReturnCode(document::ByteBuffer& buf) { - api::ReturnCode::Result result = (api::ReturnCode::Result) getInt(buf); - vespalib::stringref message = getString(buf); - return api::ReturnCode(result, message); - } - - static void putReturnCode(const api::ReturnCode& code, vespalib::GrowableByteBuffer& buf) - { - buf.putInt(code.getResult()); - buf.putString(code.getMessage()); - } - - static const uint32_t BUCKET_INFO_SERIALIZED_SIZE = sizeof(uint32_t) * 3; - static document::GlobalId getGlobalId(document::ByteBuffer& buf) { std::vector<char> buffer(getShort(buf)); for (uint32_t i=0; i<buffer.size(); ++i) { @@ -74,13 +47,6 @@ public: return document::GlobalId(&buffer[0]); } - static void putGlobalId(const document::GlobalId& gid, vespalib::GrowableByteBuffer& buf) - { - buf.putShort(document::GlobalId::LENGTH); - for (uint32_t i=0; i<document::GlobalId::LENGTH; ++i) { - buf.putByte(gid.get()[i]); - } - } static document::Document::UP getDocument(document::ByteBuffer& buf, const document::DocumentTypeRepo& repo) { uint32_t size = getInt(buf); diff --git a/storage/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp b/storage/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp index 40c2cc3b111..84b12d34e01 100644 --- a/storage/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp +++ b/storage/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp @@ -126,8 +126,8 @@ namespace { struct MetricHookWrapper : public metrics::UpdateHook { MetricUpdateHook& _hook; - MetricHookWrapper(vespalib::stringref name, MetricUpdateHook& hook) - : metrics::UpdateHook(name.data()), // Expected to point to static name + MetricHookWrapper(vespalib::stringref name, MetricUpdateHook& hook, vespalib::system_time::duration period) + : metrics::UpdateHook(name.data(), period), // Expected to point to static name _hook(hook) { } @@ -139,11 +139,11 @@ namespace { void ComponentRegisterImpl::registerUpdateHook(vespalib::stringref name, MetricUpdateHook& hook, - vespalib::duration period) + vespalib::system_time::duration period) { std::lock_guard lock(_componentLock); - auto hookPtr = std::make_unique<MetricHookWrapper>(name, hook); - _metricManager->addMetricUpdateHook(*hookPtr, vespalib::to_s(period)); + auto hookPtr = std::make_unique<MetricHookWrapper>(name, hook, period); + _metricManager->addMetricUpdateHook(*hookPtr); _hooks.emplace_back(std::move(hookPtr)); } diff --git a/storage/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h b/storage/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h index e569288ac64..43005575032 100644 --- a/storage/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h +++ b/storage/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h @@ -73,7 +73,7 @@ public: std::vector<const StatusReporter*> getStatusReporters() override; void registerMetric(metrics::Metric&) override; - void registerUpdateHook(vespalib::stringref name, MetricUpdateHook& hook, vespalib::duration period) override; + void registerUpdateHook(vespalib::stringref name, MetricUpdateHook& hook, vespalib::system_time::duration period) override; void registerShutdownListener(ShutdownListener&); }; diff --git a/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp b/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp index 314434a4c1a..c1fa2aac708 100644 --- a/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp +++ b/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp @@ -5,6 +5,7 @@ #include <vespa/storageframework/generic/clock/clock.h> #include <vespa/vespalib/util/atomic.h> #include <vespa/vespalib/util/signalhandler.h> +#include <cinttypes> #include <vespa/log/bufferedlogger.h> LOG_SETUP(".framework.thread.impl"); diff --git a/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h b/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h index 07b2dd78ed9..4319b4a0efe 100644 --- a/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h +++ b/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h @@ -4,8 +4,6 @@ #include <vespa/storageframework/generic/thread/threadpool.h> -class FastOS_ThreadPool; - namespace storage::framework { struct Clock; } diff --git a/storage/src/vespa/storageframework/generic/component/component.cpp b/storage/src/vespa/storageframework/generic/component/component.cpp index 0f08503852c..c69e59b8eba 100644 --- a/storage/src/vespa/storageframework/generic/component/component.cpp +++ b/storage/src/vespa/storageframework/generic/component/component.cpp @@ -52,7 +52,7 @@ Component::registerMetric(metrics::Metric& m) } void -Component::registerMetricUpdateHook(MetricUpdateHook& hook, vespalib::duration period) +Component::registerMetricUpdateHook(MetricUpdateHook& hook, vespalib::system_time::duration period) { assert(_metricUpdateHook.first == 0); _metricUpdateHook = std::make_pair(&hook, period); diff --git a/storage/src/vespa/storageframework/generic/component/component.h b/storage/src/vespa/storageframework/generic/component/component.h index 9a5e524e504..372559e133d 100644 --- a/storage/src/vespa/storageframework/generic/component/component.h +++ b/storage/src/vespa/storageframework/generic/component/component.h @@ -45,12 +45,12 @@ * optimize clock fetching as we see fit later. * * - A thread pool is given. This makes us able to use a thread pool. - * (Allthough currently we don't really need a thread pool, as threads - * typically live for the whole lifetime of the server. But currently we are - * forced to use a thread pool due to fastos.) Another feature of this is - * that the thread interface has built in information needed to detect - * deadlocks and report status about thread behavior, such that deadlock - * detecting and thread status can be shown without the threads themselves + * (Allthough currently we don't really need a thread pool, as + * threads typically live for the whole lifetime of the + * server. Another feature of this is that the thread interface has + * built in information needed to detect deadlocks and report + * status about thread behavior, such that deadlock detecting and + * thread status can be shown without the threads themselves * depending on how this is done. * * - A memory manager may also be provided, allowing components to request @@ -86,7 +86,7 @@ class Component : private ManagedComponent metrics::Metric* _metric; ThreadPool* _threadPool; MetricRegistrator* _metricReg; - std::pair<MetricUpdateHook*, vespalib::duration> _metricUpdateHook; + std::pair<MetricUpdateHook*, vespalib::system_time::duration> _metricUpdateHook; const Clock* _clock; // ManagedComponent implementation @@ -124,7 +124,7 @@ public: * update hook will only be called if there actually is a metric mananger * component registered in the application. */ - void registerMetricUpdateHook(MetricUpdateHook&, vespalib::duration period); + void registerMetricUpdateHook(MetricUpdateHook&, vespalib::system_time::duration period); /** Get the name of the component. Must be a unique name. */ [[nodiscard]] const vespalib::string& getName() const override { return _name; } diff --git a/storage/src/vespa/storageframework/generic/metric/metricregistrator.h b/storage/src/vespa/storageframework/generic/metric/metricregistrator.h index 6daca1213a8..bea43fcfb6b 100644 --- a/storage/src/vespa/storageframework/generic/metric/metricregistrator.h +++ b/storage/src/vespa/storageframework/generic/metric/metricregistrator.h @@ -24,7 +24,7 @@ struct MetricRegistrator { virtual ~MetricRegistrator() = default; virtual void registerMetric(metrics::Metric&) = 0; - virtual void registerUpdateHook(vespalib::stringref name, MetricUpdateHook& hook, vespalib::duration period) = 0; + virtual void registerUpdateHook(vespalib::stringref name, MetricUpdateHook& hook, vespalib::system_time::duration period) = 0; }; } diff --git a/storageserver/CMakeLists.txt b/storageserver/CMakeLists.txt index ee3335c4921..a2f9d0b776e 100644 --- a/storageserver/CMakeLists.txt +++ b/storageserver/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos storage streamingvisitors diff --git a/streamingvisitors/CMakeLists.txt b/streamingvisitors/CMakeLists.txt index 2c7f01ddf37..fede7087d8d 100644 --- a/streamingvisitors/CMakeLists.txt +++ b/streamingvisitors/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog storage config_cloudconfig diff --git a/vbench/CMakeLists.txt b/vbench/CMakeLists.txt index 3fb5df8cd20..e78913262be 100644 --- a/vbench/CMakeLists.txt +++ b/vbench/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalib LIBS diff --git a/vdslib/CMakeLists.txt b/vdslib/CMakeLists.txt index 0f8144b99e9..1276323f83b 100644 --- a/vdslib/CMakeLists.txt +++ b/vdslib/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib config_cloudconfig diff --git a/vdstestlib/CMakeLists.txt b/vdstestlib/CMakeLists.txt index d0a921672c2..7f478989332 100644 --- a/vdstestlib/CMakeLists.txt +++ b/vdstestlib/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalib TESTS diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java index cf46cad57b1..21c8f4ddd31 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.athenz.api.NToken; import com.yahoo.vespa.athenz.api.ZToken; import com.yahoo.vespa.athenz.client.ErrorHandler; import com.yahoo.vespa.athenz.client.common.ClientBase; +import com.yahoo.vespa.athenz.client.zms.bindings.AccessResponseEntity; import com.yahoo.vespa.athenz.client.zts.bindings.AccessTokenResponseEntity; import com.yahoo.vespa.athenz.client.zts.bindings.AwsTemporaryCredentialsResponseEntity; import com.yahoo.vespa.athenz.client.zts.bindings.IdentityRefreshRequestEntity; @@ -221,6 +222,19 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient { }); } + @Override + public boolean hasAccess(AthenzResourceName resource, String action, AthenzIdentity identity) { + URI uri = ztsUrl.resolve(String.format("access/%s/%s?principal=%s", + action, resource.toResourceNameString(), identity.getFullName())); + HttpUriRequest request = RequestBuilder.get() + .setUri(uri) + .build(); + return execute(request, response -> { + AccessResponseEntity result = readEntity(response, AccessResponseEntity.class); + return result.granted; + }); + } + private InstanceIdentity getInstanceIdentity(HttpResponse response) throws IOException { InstanceIdentityCredentials entity = readEntity(response, InstanceIdentityCredentials.class); return entity.getServiceToken() != null diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java index c4be6d8ced7..eade6229123 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java @@ -5,6 +5,7 @@ import com.yahoo.security.Pkcs10Csr; import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.AwsRole; import com.yahoo.vespa.athenz.api.AwsTemporaryCredentials; @@ -187,5 +188,16 @@ public interface ZtsClient extends AutoCloseable { */ AwsTemporaryCredentials getAwsTemporaryCredentials(AthenzDomain athenzDomain, AwsRole awsRole, Duration duration, String externalId); + /** + * Check access to resource for a given principal + * + * @param resource The resource to verify access to + * @param action Action to verify + * @param identity Principal that requests access + * @return <code>true</code> if access is allowed, <code>false</code> otherwise + */ + boolean hasAccess(AthenzResourceName resource, String action, AthenzIdentity identity); + void close(); + } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java index 9b7b666e353..2d77d2ceda1 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java @@ -4,8 +4,10 @@ package com.yahoo.vespa.athenz.identityprovider.api; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; +import com.yahoo.vespa.athenz.utils.AthenzIdentities; import java.io.IOException; import java.io.InputStream; @@ -58,6 +60,8 @@ public class EntityBindingsMapper { entity.ipAddresses(), IdentityType.fromId(entity.identityType()), Optional.ofNullable(entity.clusterType()).map(ClusterType::from).orElse(null), + entity.ztsUrl(), + Optional.ofNullable(entity.serviceIdentity()).map(AthenzIdentities::from).orElse(null), entity.unknownAttributes()); } @@ -74,6 +78,8 @@ public class EntityBindingsMapper { model.ipAddresses(), model.identityType().id(), Optional.ofNullable(model.clusterType()).map(ClusterType::toConfigValue).orElse(null), + model.ztsUrl(), + Optional.ofNullable(model.serviceIdentity()).map(AthenzIdentity::getFullName).orElse(null), model.unknownAttributes()); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java index b18ff238b07..de78d81cd1b 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java @@ -1,9 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.identityprovider.api; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; +import java.net.URL; import java.time.Instant; +import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -16,23 +19,36 @@ import java.util.Set; public record SignedIdentityDocument(String signature, int signingKeyVersion, VespaUniqueInstanceId providerUniqueId, AthenzService providerService, int documentVersion, String configServerHostname, String instanceHostname, Instant createdAt, Set<String> ipAddresses, - IdentityType identityType, ClusterType clusterType, Map<String, Object> unknownAttributes) { + IdentityType identityType, ClusterType clusterType, String ztsUrl, + AthenzIdentity serviceIdentity, Map<String, Object> unknownAttributes) { public SignedIdentityDocument { ipAddresses = Set.copyOf(ipAddresses); - unknownAttributes = Map.copyOf(unknownAttributes); + + Map<String, Object> nonNull = new HashMap<>(); + unknownAttributes.forEach((key, value) -> { + if (value != null) nonNull.put(key, value); + }); + // Map.copyOf() does not allow null values + unknownAttributes = Map.copyOf(nonNull); } public SignedIdentityDocument(String signature, int signingKeyVersion, VespaUniqueInstanceId providerUniqueId, AthenzService providerService, int documentVersion, String configServerHostname, String instanceHostname, Instant createdAt, Set<String> ipAddresses, - IdentityType identityType, ClusterType clusterType) { + IdentityType identityType, ClusterType clusterType, String ztsUrl, AthenzIdentity serviceIdentity) { this(signature, signingKeyVersion, providerUniqueId, providerService, documentVersion, configServerHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, Map.of()); + instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity, Map.of()); } - public static final int DEFAULT_DOCUMENT_VERSION = 2; + public static final int DEFAULT_DOCUMENT_VERSION = 3; public boolean outdated() { return documentVersion < DEFAULT_DOCUMENT_VERSION; } + public SignedIdentityDocument withServiceIdentity(AthenzIdentity identity) { + return new SignedIdentityDocument(signature, signingKeyVersion, providerUniqueId, providerService, documentVersion, configServerHostname, instanceHostname, createdAt, + ipAddresses, identityType, clusterType, ztsUrl, identity); + } + + } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java index c37dd2f9147..fc0dff3b97b 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.athenz.identityprovider.api.bindings; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import java.time.Instant; @@ -14,10 +15,11 @@ import java.util.Set; /** * @author bjorncs */ +@JsonInclude(JsonInclude.Include.NON_NULL) public record SignedIdentityDocumentEntity( String signature, int signingKeyVersion, String providerUniqueId, String providerService, int documentVersion, String configServerHostname, String instanceHostname, Instant createdAt, Set<String> ipAddresses, - String identityType, String clusterType, Map<String, Object> unknownAttributes) { + String identityType, String clusterType, String ztsUrl, String serviceIdentity, Map<String, Object> unknownAttributes) { @JsonCreator public SignedIdentityDocumentEntity(@JsonProperty("signature") String signature, @@ -30,9 +32,11 @@ public record SignedIdentityDocumentEntity( @JsonProperty("created-at") Instant createdAt, @JsonProperty("ip-addresses") Set<String> ipAddresses, @JsonProperty("identity-type") String identityType, - @JsonProperty("cluster-type") String clusterType) { + @JsonProperty("cluster-type") String clusterType, + @JsonProperty("zts-url") String ztsUrl, + @JsonProperty("service-identity") String serviceIdentity) { this(signature, signingKeyVersion, providerUniqueId, providerService, documentVersion, configServerHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, new HashMap<>()); + instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity, new HashMap<>()); } @JsonProperty("signature") @Override public String signature() { return signature; } @@ -46,6 +50,8 @@ public record SignedIdentityDocumentEntity( @JsonProperty("ip-addresses") @Override public Set<String> ipAddresses() { return ipAddresses; } @JsonProperty("identity-type") @Override public String identityType() { return identityType; } @JsonProperty("cluster-type") @Override public String clusterType() { return clusterType; } + @JsonProperty("zts-url") @Override public String ztsUrl() { return ztsUrl; } + @JsonProperty("service-identity") @Override public String serviceIdentity() { return serviceIdentity; } @JsonAnyGetter @Override public Map<String, Object> unknownAttributes() { return unknownAttributes; } @JsonAnySetter public void set(String name, Object value) { unknownAttributes.put(name, value); } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java index 14d06fe83f2..019f73fc6bf 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.athenz.identityprovider.client; import com.yahoo.security.SignatureUtils; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; @@ -18,6 +19,7 @@ import java.util.Base64; import java.util.Set; import java.util.TreeSet; +import static com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION; import static java.nio.charset.StandardCharsets.UTF_8; /** @@ -35,13 +37,15 @@ public class IdentityDocumentSigner { Instant createdAt, Set<String> ipAddresses, IdentityType identityType, - PrivateKey privateKey) { + PrivateKey privateKey, + AthenzIdentity serviceIdentity) { try { Signature signer = SignatureUtils.createSigner(privateKey); signer.initSign(privateKey); writeToSigner( signer, providerUniqueId, providerService, configServerHostname, instanceHostname, createdAt, ipAddresses, identityType); + writeToSigner(signer, serviceIdentity); byte[] signature = signer.sign(); return Base64.getEncoder().encodeToString(signature); } catch (GeneralSecurityException e) { @@ -56,6 +60,9 @@ public class IdentityDocumentSigner { writeToSigner( signer, doc.providerUniqueId(), doc.providerService(), doc.configServerHostname(), doc.instanceHostname(), doc.createdAt(), doc.ipAddresses(), doc.identityType()); + if (doc.documentVersion() >= DEFAULT_DOCUMENT_VERSION) { + writeToSigner(signer, doc.serviceIdentity()); + } return signer.verify(Base64.getDecoder().decode(doc.signature())); } catch (GeneralSecurityException e) { throw new RuntimeException(e); @@ -82,4 +89,8 @@ public class IdentityDocumentSigner { } signer.update(identityType.id().getBytes(UTF_8)); } + + private static void writeToSigner(Signature signer, AthenzIdentity serviceIdentity) throws SignatureException{ + signer.update(serviceIdentity.getFullName().getBytes(UTF_8)); + } } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java index f8c119190a6..2a68f6fd231 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java @@ -30,6 +30,7 @@ class EntityBindingsMapperTest { "ip-addresses": [], "identity-type": "node", "cluster-type": "admin", + "zts-url": "https://zts.url/", "unknown-string": "string-value", "unknown-object": { "member-in-unknown-object": 123 } } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java index 0b8ff4277f1..ff85cb79f02 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java @@ -3,11 +3,13 @@ package com.yahoo.vespa.athenz.identityprovider.client; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.ClusterType; import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; +import com.yahoo.vespa.athenz.utils.AthenzIdentities; import org.junit.jupiter.api.Test; import java.security.KeyPair; @@ -36,37 +38,54 @@ public class IdentityDocumentSignerTest { private static final Instant createdAt = Instant.EPOCH; private static final HashSet<String> ipAddresses = new HashSet<>(Arrays.asList("1.2.3.4", "::1")); private static final ClusterType clusterType = ClusterType.CONTAINER; + private static final String ztsUrl = "https://foo"; + private static final AthenzIdentity serviceIdentity = new AthenzService("vespa", "node"); @Test void generates_and_validates_signature() { IdentityDocumentSigner signer = new IdentityDocumentSigner(); String signature = signer.generateSignature(id, providerService, configserverHostname, instanceHostname, createdAt, - ipAddresses, identityType, keyPair.getPrivate()); + ipAddresses, identityType, keyPair.getPrivate(), serviceIdentity); SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument( signature, KEY_VERSION, id, providerService, DEFAULT_DOCUMENT_VERSION, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType); + instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity); assertTrue(signer.hasValidSignature(signedIdentityDocument, keyPair.getPublic())); } @Test - void ignores_cluster_type() { + void ignores_cluster_type_and_zts_url() { IdentityDocumentSigner signer = new IdentityDocumentSigner(); String signature = signer.generateSignature(id, providerService, configserverHostname, instanceHostname, createdAt, - ipAddresses, identityType, keyPair.getPrivate()); + ipAddresses, identityType, keyPair.getPrivate(), serviceIdentity); - var docWithoutClusterType = new SignedIdentityDocument( + var docWithoutIgnoredFields = new SignedIdentityDocument( signature, KEY_VERSION, id, providerService, DEFAULT_DOCUMENT_VERSION, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, null); - var docWithClusterType = new SignedIdentityDocument( + instanceHostname, createdAt, ipAddresses, identityType, null, null, serviceIdentity); + var docWithIgnoredFields = new SignedIdentityDocument( signature, KEY_VERSION, id, providerService, DEFAULT_DOCUMENT_VERSION, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType); + instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity); + + assertTrue(signer.hasValidSignature(docWithoutIgnoredFields, keyPair.getPublic())); + assertEquals(docWithIgnoredFields.signature(), docWithoutIgnoredFields.signature()); + } + + @Test + void validates_signature_for_new_and_old_versions() { + IdentityDocumentSigner signer = new IdentityDocumentSigner(); + String signature = + signer.generateSignature(id, providerService, configserverHostname, instanceHostname, createdAt, + ipAddresses, identityType, keyPair.getPrivate(), serviceIdentity); + + SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument( + signature, KEY_VERSION, id, providerService, DEFAULT_DOCUMENT_VERSION, configserverHostname, + instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity); + + assertTrue(signer.hasValidSignature(signedIdentityDocument, keyPair.getPublic())); - assertTrue(signer.hasValidSignature(docWithoutClusterType, keyPair.getPublic())); - assertEquals(docWithClusterType.signature(), docWithoutClusterType.signature()); } }
\ No newline at end of file diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 2cd7c6247dd..02f75c19907 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -14,21 +14,20 @@ com.amazonaws:aws-java-sdk-ssm:1.12.331 com.amazonaws:aws-java-sdk-sts:1.12.331 com.amazonaws:jmespath-java:1.12.331 com.auth0:java-jwt:3.10.0 -com.fasterxml.jackson.core:jackson-annotations:2.13.4 -com.fasterxml.jackson.core:jackson-core:2.13.4 -com.fasterxml.jackson.core:jackson-databind:2.13.4.2 -com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.12.6 -com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.13.4 -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.13.4 -com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.13.4 -com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.13.4 -com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.13.4 +com.fasterxml.jackson.core:jackson-annotations:2.14.2 +com.fasterxml.jackson.core:jackson-core:2.14.2 +com.fasterxml.jackson.core:jackson-databind:2.14.2 +com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.14.2 +com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.14.2 +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.14.2 +com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.14.2 +com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.14.2 +com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.14.2 com.github.spotbugs:spotbugs-annotations:3.1.9 com.google.code.findbugs:jsr305:3.0.2 com.google.errorprone:error_prone_annotations:2.18.0 com.google.guava:failureaccess:1.0.1 com.google.guava:guava:27.1-jre -com.google.inject:guice:4.2.3 com.google.inject:guice:4.2.3:no_aop com.google.j2objc:j2objc-annotations:1.1 com.google.protobuf:protobuf-java:3.21.7 @@ -49,7 +48,7 @@ com.yahoo.athenz:athenz-zts-core:1.10.54 com.yahoo.rdl:rdl-java:1.5.4 commons-cli:commons-cli:1.5.0 commons-codec:commons-codec:1.15 -commons-fileupload:commons-fileupload:1.4 +commons-fileupload:commons-fileupload:1.5 commons-io:commons-io:2.11.0 commons-logging:commons-logging:1.2 io.airlift:airline:0.9 @@ -87,9 +86,9 @@ org.apache.commons:commons-csv:1.8 org.apache.commons:commons-exec:1.3 org.apache.commons:commons-lang3:3.12.0 org.apache.commons:commons-math3:3.6.1 -org.apache.curator:curator-client:5.3.0 -org.apache.curator:curator-framework:5.3.0 -org.apache.curator:curator-recipes:5.3.0 +org.apache.curator:curator-client:5.4.0 +org.apache.curator:curator-framework:5.4.0 +org.apache.curator:curator-recipes:5.4.0 org.apache.felix:org.apache.felix.framework:7.0.1 org.apache.felix:org.apache.felix.log:1.0.1 org.apache.httpcomponents:httpclient:4.5.14 @@ -221,11 +220,12 @@ xml-apis:xml-apis:1.4.01 com.github.luben:zstd-jni:1.5.2-1 com.github.tomakehurst:wiremock-jre8-standalone:2.35.0 com.google.guava:guava-testlib:27.1-jre +com.google.inject:guice:4.2.3 com.google.jimfs:jimfs:1.2 junit:junit:4.13.2 net.bytebuddy:byte-buddy:1.11.19 net.bytebuddy:byte-buddy-agent:1.11.19 -org.apache.curator:curator-test:5.3.0 +org.apache.curator:curator-test:5.4.0 org.assertj:assertj-core:3.11.1 org.checkerframework:checker-qual:3.30.0 org.cthul:cthul-matchers:1.0 diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java index 0f3531720ae..d753f3d9e73 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -448,7 +448,7 @@ public class DocumentGenMojo extends AbstractMojo { out.write(ind(1)+"@Override protected boolean isGenerated() { return true; }\n\n"); Collection<Field> allUniqueFields = getAllUniqueFields(multiExtends, docType.getAllFields()); - exportExtendedStructTypeGetter(className, docType.getName(), allUniqueFields, docType.getFieldSets(), + exportExtendedStructTypeGetter(className, docType.getName(), docType.getInherited(), allUniqueFields, docType.getFieldSets(), docType.getImportedFieldNames(), out, 1, "getDocumentType", "com.yahoo.document.DocumentType"); exportCopyConstructor(className, out, 1, true); @@ -612,14 +612,17 @@ public class DocumentGenMojo extends AbstractMojo { out.write(ind(ind) + "importedFieldNames.add(\"" + importedField + "\");\n"); } } - private static void exportExtendedStructTypeGetter(String className, String name, Collection<Field> fields, Set<FieldSet> fieldSets, - Set<String> importedFieldNames, Writer out, int ind, String methodName, String retType) throws IOException { + private static void exportExtendedStructTypeGetter(String className, String name, Collection<NewDocumentType> parentTypes, + Collection<Field> fields, Set<FieldSet> fieldSets, + Set<String> importedFieldNames, Writer out, int ind, + String methodName, String retType) throws IOException { out.write(ind(ind)+"private static "+retType+" "+methodName+"() {\n"); + String bodyIndent = ind(ind + 1); if (importedFieldNames != null) { exportImportedFields(importedFieldNames, out, ind + 1); - out.write(ind(ind+1)+retType+" ret = new "+retType+"(\"" + name + "\", importedFieldNames);\n"); + out.write(bodyIndent+retType+" ret = new "+retType+"(\"" + name + "\", importedFieldNames);\n"); } else { - out.write(ind(ind+1)+retType+" ret = new "+retType+"(\""+name+"\");\n"); + out.write(bodyIndent+retType+" ret = new "+retType+"(\""+name+"\");\n"); } for (Field f : fields) { if (f.getDataType().equals(DataType.STRING)) { @@ -631,8 +634,13 @@ public class DocumentGenMojo extends AbstractMojo { if (fieldSets != null) { exportFieldSetDefinition(fieldSets, out, ind+1); } + for (NewDocumentType parentType : parentTypes) { + if (!parentType.getName().equals("document")) { + out.write("%sret.inherit(%s.type);\n".formatted(bodyIndent, className(parentType.getName()))); + } + } - out.write(ind(ind+1)+"return ret;\n"); + out.write(bodyIndent+"return ret;\n"); out.write(ind(ind)+"}\n\n"); } @@ -762,7 +770,9 @@ public class DocumentGenMojo extends AbstractMojo { ind(ind+2)+"super("+structClassName+".type);\n" + ind(ind+1)+"}\n\n"); exportCopyConstructor(structClassName, out, ind+1, false); - exportExtendedStructTypeGetter(structClassName, structType.getName(), structType.getFields(), null, null, out, ind+1, "getStructType", "com.yahoo.document.StructDataType"); + exportExtendedStructTypeGetter(structClassName, structType.getName(), List.of(), + structType.getFields(), null, null, out, ind+1, "getStructType", + "com.yahoo.document.StructDataType"); exportAssign(structType, structClassName, out, ind+1); exportFieldsAndAccessors(structClassName, structType.getFields(), out, ind+1, true); diff --git a/vespabase/src/vespa-configserver.service.in b/vespabase/src/vespa-configserver.service.in index f160c2ea8ad..7113ea501c1 100644 --- a/vespabase/src/vespa-configserver.service.in +++ b/vespabase/src/vespa-configserver.service.in @@ -5,6 +5,7 @@ After=network.target [Service] Type=forking +User=vespa PIDFile=@CMAKE_INSTALL_PREFIX@/var/run/configserver.pid ExecStart=@CMAKE_INSTALL_PREFIX@/bin/vespa-start-configserver ExecStop=@CMAKE_INSTALL_PREFIX@/bin/vespa-stop-configserver diff --git a/vespabase/src/vespa.service.in b/vespabase/src/vespa.service.in index e10724ef736..707d67ff98f 100644 --- a/vespabase/src/vespa.service.in +++ b/vespabase/src/vespa.service.in @@ -5,6 +5,7 @@ After=network.target [Service] Type=forking +User=vespa PIDFile=@CMAKE_INSTALL_PREFIX@/var/run/sentinel.pid ExecStart=@CMAKE_INSTALL_PREFIX@/bin/vespa-start-services ExecStop=@CMAKE_INSTALL_PREFIX@/bin/vespa-stop-services diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java index 438248f31a7..b44fe82a303 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -188,16 +188,12 @@ class ClientFeederV3 { outstandingOperations.incrementAndGet(); updateOpsPerSec(); log(Level.FINE, "Sent message successfully, document id: ", message.get().getOperationId()); - } else if (!result.getError().isFatal()) { - repliesFromOldMessages.add(createOperationStatus(message.get().getOperationId(), - result.getError().getMessage(), - ErrorCode.TRANSIENT_ERROR, - message.get().getMessage())); } else { - repliesFromOldMessages.add(createOperationStatus(message.get().getOperationId(), - result.getError().getMessage(), - ErrorCode.ERROR, - message.get().getMessage())); + var err = result.getError(); + var msg = message.get(); + repliesFromOldMessages.add( + createOperationStatus( + msg.getOperationId(), err.getMessage(), ErrorCode.fromBusError(err), msg.getMessage())); } } } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ErrorCode.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ErrorCode.java index f819ecccbb1..90c6ffd042d 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ErrorCode.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ErrorCode.java @@ -1,6 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.server; +import com.yahoo.messagebus.Error; + +import java.util.Collection; +import java.util.Set; + /** * Return types for the server. * @@ -14,6 +19,15 @@ enum ErrorCode { TRANSIENT_ERROR(false, true), END_OF_FEED(true, true); + private static final Collection<Integer> MBUS_FATALS_HANDLED_AS_TRANSIENT = Set.of( + com.yahoo.messagebus.ErrorCode.SEND_QUEUE_CLOSED, + com.yahoo.messagebus.ErrorCode.ILLEGAL_ROUTE, + com.yahoo.messagebus.ErrorCode.NO_SERVICES_FOR_ROUTE, + com.yahoo.messagebus.ErrorCode.NETWORK_ERROR, + com.yahoo.messagebus.ErrorCode.SEQUENCE_ERROR, + com.yahoo.messagebus.ErrorCode.NETWORK_SHUTDOWN, + com.yahoo.messagebus.ErrorCode.TIMEOUT); + private final boolean success; private final boolean _transient; @@ -30,4 +44,9 @@ enum ErrorCode { return _transient; } + static ErrorCode fromBusError(Error mbusError) { + return mbusError.isFatal() && !MBUS_FATALS_HANDLED_AS_TRANSIENT.contains(mbusError.getCode()) + ? ERROR : TRANSIENT_ERROR; + } + } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java index 377e91f6490..b504de64c63 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java @@ -50,7 +50,7 @@ public class FeedReplyReader implements ReplyHandler { DocumentOperationStatus status = DocumentOperationStatus.fromMessageBusErrorCodes(reply.getErrorCodes()); metricsHelper.reportFailure(type, status); metric.add(MetricNames.FAILED, 1, null); - enqueue(context, reply.getError(0).getMessage(), ErrorCode.ERROR, false, reply.getTrace()); + enqueue(context, reply.getError(0).getMessage(), ErrorCode.fromBusError(reply.getError(0)), false, reply.getTrace()); } else { metricsHelper.reportSuccessful(type, latencyInSeconds); if ( ! conditionMet) diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java index fc74cb6d899..c2dea5e563b 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java @@ -35,42 +35,46 @@ import java.util.logging.Logger; @SuppressWarnings("deprecation") public class StdOutVisitorHandler extends VdsVisitHandler { - private static final Logger log = Logger.getLogger( - StdOutVisitorHandler.class.getName()); - private final boolean printIds; - private final boolean indentXml; - private final int processTimeMilliSecs; - private final PrintStream out; - private final boolean jsonOutput; - private final boolean tensorShortForm; - private final boolean tensorDirectValues; + private static final Logger log = Logger.getLogger(StdOutVisitorHandler.class.getName()); - private final VisitorDataHandler dataHandler; + public enum OutputFormat { + JSONL, + JSON, + XML // Deprecated + } + + // Explicitly _not_ a record since we want the fields to be mutable when building. + public static class Params { + boolean printIds = false; + boolean indentXml = false; + boolean showProgress = false; + boolean showStatistics = false; + boolean doStatistics = false; + boolean abortOnClusterDown = false; + int processTimeMilliSecs = 0; + OutputFormat outputFormat = OutputFormat.JSON; + boolean tensorShortForm = false; // TODO Vespa 9: change default to true + boolean tensorDirectValues = false; // TODO Vespa 9: change default to true + boolean nullRender = false; - public StdOutVisitorHandler(boolean printIds, boolean indentXml, - boolean showProgress, boolean showStatistics, boolean doStatistics, - boolean abortOnClusterDown, int processtime, boolean jsonOutput, - boolean tensorShortForm, - boolean tensorDirectValues) - { - this(printIds, indentXml, showProgress, showStatistics, doStatistics, abortOnClusterDown, processtime, - jsonOutput, tensorShortForm, tensorDirectValues, createStdOutPrintStream()); + boolean usesJson() { + return outputFormat == OutputFormat.JSON || outputFormat == OutputFormat.JSONL; + } } - StdOutVisitorHandler(boolean printIds, boolean indentXml, - boolean showProgress, boolean showStatistics, boolean doStatistics, - boolean abortOnClusterDown, int processtime, boolean jsonOutput, - boolean tensorShortForm, boolean tensorDirectValues, PrintStream out) - { - super(showProgress, showStatistics, abortOnClusterDown); - this.printIds = printIds; - this.indentXml = indentXml; - this.processTimeMilliSecs = processtime; - this.jsonOutput = jsonOutput; - this.tensorShortForm = tensorShortForm; - this.tensorDirectValues = tensorDirectValues; + private final Params params; + private final PrintStream out; + private final VisitorDataHandler dataHandler; + + public StdOutVisitorHandler(Params params, PrintStream out) { + super(params.showProgress, params.showStatistics, params.abortOnClusterDown); + this.params = params; this.out = out; - this.dataHandler = new DataHandler(doStatistics); + this.dataHandler = new DataHandler(params.doStatistics); + } + + public StdOutVisitorHandler(Params params) { + this(params, createStdOutPrintStream()); } private static PrintStream createStdOutPrintStream() { @@ -128,9 +132,9 @@ public class StdOutVisitorHandler extends VdsVisitHandler { @Override public void onMessage(Message m, AckToken token) { - if (processTimeMilliSecs > 0) { + if (params.processTimeMilliSecs > 0) { try { - Thread.sleep(processTimeMilliSecs); + Thread.sleep(params.processTimeMilliSecs); } catch (InterruptedException e) {} } @@ -154,20 +158,22 @@ public class StdOutVisitorHandler extends VdsVisitHandler { @Override public void onDocument(Document doc, long timestamp) { try { + if (params.nullRender) { + return; + } if (lastLineIsProgress) { System.err.print('\r'); } - if (printIds) { + if (params.printIds) { out.print(doc.getId()); out.print(" (Last modified at "); out.println(timestamp + ")"); } else { - if (jsonOutput) { + if (params.usesJson()) { writeJsonDocument(doc); } else { - out.print(doc.toXML( - indentXml ? " " : "")); + out.print(doc.toXML(params.indentXml ? " " : "")); } } } catch (Exception e) { @@ -179,20 +185,23 @@ public class StdOutVisitorHandler extends VdsVisitHandler { private void writeJsonDocument(Document doc) throws IOException { writeFeedStartOrRecordSeparator(); - out.write(JsonWriter.toByteArray(doc, tensorShortForm, tensorDirectValues)); + out.write(JsonWriter.toByteArray(doc, params.tensorShortForm, params.tensorDirectValues)); } @Override public void onRemove(DocumentId docId) { try { + if (params.nullRender) { + return; + } if (lastLineIsProgress) { System.err.print('\r'); } - if (printIds) { + if (params.printIds) { out.println(docId + " (Removed)"); } else { - if (jsonOutput) { + if (params.usesJson()) { writeJsonDocumentRemove(docId); } else { XmlStream stream = new XmlStream(); @@ -218,10 +227,12 @@ public class StdOutVisitorHandler extends VdsVisitHandler { private void writeFeedStartOrRecordSeparator() { if (first) { - out.println("["); + if (params.outputFormat == OutputFormat.JSON) { + out.println("["); + } first = false; } else { - out.println(","); + out.println((params.outputFormat == OutputFormat.JSON) ? "," : ""); } } @@ -259,7 +270,7 @@ public class StdOutVisitorHandler extends VdsVisitHandler { @Override public synchronized void onDone() { - if (jsonOutput && !printIds) { + if ((params.outputFormat == OutputFormat.JSON) && !params.printIds && !params.nullRender) { if (first) { out.print('['); } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java index a6e34055fbd..8b919f7e9ea 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java @@ -23,9 +23,12 @@ import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; -import java.io.*; -import java.nio.charset.Charset; +import java.io.IOException; +import java.io.PrintStream; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; import java.util.Map; import java.util.stream.Collectors; @@ -334,9 +337,15 @@ public class VdsVisit { .hasArg(false) .build()); + options.addOption(Option.builder() + .longOpt("jsonl") + .desc("Output documents as JSONL (JSON Lines format)") + .hasArg(false) + .build()); + options.addOption(Option.builder("x") .longOpt("xmloutput") - .desc("Output documents as XML") + .desc("Output documents as XML (deprecated)") .hasArg(false) .build()); @@ -355,6 +364,30 @@ public class VdsVisit { .hasArg(false) .build()); + options.addOption(Option.builder() + .longOpt("slices") + .desc("Split the document corpus into this number of independent slices. " + + "This lets multiple, concurrent series of visitors advance the same logical " + + "visit independently, by specifying a different --sliceid for each.") + .hasArg(true) + .type(Number.class) + .build()); + + options.addOption(Option.builder() + .longOpt("sliceid") + .desc("The slice number of the visit represented by this visitor. " + + "This number must be non-negative and less than the number of slices specified for the visit.") + .hasArg(true) + .type(Number.class) + .build()); + + options.addOption(Option.builder() + .longOpt("nullrender") + .desc("Process documents, but do not render any output. Overrides all other output options. " + + "Used to benchmark whether document rendering is the bottleneck when processing documents.") + .hasArg(false) + .build()); + return options; } @@ -370,8 +403,12 @@ public class VdsVisit { private int processTime = 0; private int fullTimeout = 7 * 24 * 60 * 60 * 1000; private boolean jsonOutput = true; + private boolean jsonLinesOutput = false; private boolean tensorShortForm = false; // TODO Vespa 9: change default to true private boolean tensorDirectValues = false; // TODO Vespa 9: change default to true + private boolean nullRender = false; + private int slices = 1; + private int sliceId = 0; public VisitorParameters getVisitorParameters() { return visitorParameters; @@ -437,10 +474,32 @@ public class VdsVisit { this.processTime = processTime; } + public boolean jsonOutput() { + return jsonOutput; + } + public void setJsonOutput(boolean jsonOutput) { this.jsonOutput = jsonOutput; } + public boolean jsonLinesOutput() { + return jsonLinesOutput; + } + + public void setJsonLinesOutput(boolean jsonLinesOutput) { + this.jsonLinesOutput = jsonLinesOutput; + } + + public StdOutVisitorHandler.OutputFormat stdOutHandlerOutputFormat() { + if (jsonLinesOutput) { + return StdOutVisitorHandler.OutputFormat.JSONL; + } else if (jsonOutput) { + return StdOutVisitorHandler.OutputFormat.JSON; + } else { + return StdOutVisitorHandler.OutputFormat.XML; + } + } + public boolean tensorShortForm() { return tensorShortForm; } @@ -457,6 +516,38 @@ public class VdsVisit { this.tensorDirectValues = tensorDirectValues; } + public boolean nullRender() { + return nullRender; + } + + public void setNullRender(boolean nullRender) { + this.nullRender = nullRender; + } + + public int slices() { + return slices; + } + + public void setSlices(int slices) { + this.slices = slices; + } + + public int sliceId() { + return sliceId; + } + + public void setSliceId(int sliceId) { + this.sliceId = sliceId; + } + + } + + private static int optionAsInt(CommandLine cmdLine, String optName) throws org.apache.commons.cli.ParseException { + return ((Number)cmdLine.getParsedOptionValue(optName)).intValue(); + } + + private static long optionAsLong(CommandLine cmdLine, String optName) throws org.apache.commons.cli.ParseException { + return ((Number)cmdLine.getParsedOptionValue(optName)).longValue(); } protected static class ArgumentParser { @@ -485,10 +576,10 @@ public class VdsVisit { params.setBucketSpace(line.getOptionValue("bucketspace")); } if (line.hasOption("f")) { - params.setFromTimestamp(((Number) line.getParsedOptionValue("f")).longValue()); + params.setFromTimestamp(optionAsLong(line, "f")); } if (line.hasOption("t")) { - params.setToTimestamp(((Number) line.getParsedOptionValue("t")).longValue()); + params.setToTimestamp(optionAsLong(line, "t")); } if (line.hasOption("e")) { throw new IllegalArgumentException("Headers only option has been removed."); @@ -502,10 +593,10 @@ public class VdsVisit { params.visitInconsistentBuckets(true); } if (line.hasOption("m")) { - params.setMaxPending(((Number) line.getParsedOptionValue("m")).intValue()); + params.setMaxPending(optionAsInt(line, "m")); } if (line.hasOption("b")) { - params.setMaxBucketsPerVisitor(((Number) line.getParsedOptionValue("b")).intValue()); + params.setMaxBucketsPerVisitor(optionAsInt(line, "b")); } if (line.hasOption("i")) { allParams.setPrintIdsOnly(true); @@ -515,11 +606,11 @@ public class VdsVisit { params.setResumeFileName(line.getOptionValue("p")); } if (line.hasOption("o")) { - allParams.setFullTimeout(((Number) line.getParsedOptionValue("o")).intValue()); + allParams.setFullTimeout(optionAsInt(line, "o")); params.setTimeoutMs(allParams.getFullTimeout()); } if (line.hasOption("u")) { - params.setTimeoutMs(((Number) line.getParsedOptionValue("u")).intValue()); + params.setTimeoutMs(optionAsInt(line, "u")); } if (line.hasOption("visitlibrary")) { params.setVisitorLibrary(line.getOptionValue("visitlibrary")); @@ -550,13 +641,13 @@ public class VdsVisit { allParams.setAbortOnClusterDown(true); } if (line.hasOption("processtime")) { - allParams.setProcessTime(((Number) line.getParsedOptionValue("processtime")).intValue()); + allParams.setProcessTime(optionAsInt(line, "processtime")); } if (line.hasOption("maxtotalhits")) { - params.setMaxTotalHits(((Number)line.getParsedOptionValue("maxtotalhits")).intValue()); + params.setMaxTotalHits(optionAsLong(line, "maxtotalhits")); } if (line.hasOption("tracelevel")) { - params.setTraceLevel(((Number)line.getParsedOptionValue("tracelevel")).intValue()); + params.setTraceLevel(optionAsInt(line, "tracelevel")); } if (line.hasOption("priority")) { try { @@ -576,7 +667,7 @@ public class VdsVisit { } if (line.hasOption("maxpendingsuperbuckets")) { StaticThrottlePolicy throttlePolicy = new StaticThrottlePolicy(); - throttlePolicy.setMaxPendingCount(((Number)line.getParsedOptionValue("maxpendingsuperbuckets")).intValue()); + throttlePolicy.setMaxPendingCount(optionAsInt(line, "maxpendingsuperbuckets")); params.setThrottlePolicy(throttlePolicy); } if (line.hasOption("shorttensors")) { @@ -585,13 +676,38 @@ public class VdsVisit { if (line.hasOption("tensorvalues")) { allParams.setTensorDirectValues(true); } + if (line.hasOption("nullrender")) { + allParams.setNullRender(true); + } + if (line.hasOption("slices") != line.hasOption("sliceid")) { + throw new IllegalArgumentException("Both --slices and --sliceid must be specified when visiting with slicing"); + } + if (line.hasOption("slices")) { + allParams.setSlices(optionAsInt(line, "slices")); + allParams.setSliceId(optionAsInt(line, "sliceid")); + } boolean jsonOutput = line.hasOption("jsonoutput"); - boolean xmlOutput = line.hasOption("xmloutput"); - if (jsonOutput && xmlOutput) { - throw new IllegalArgumentException("Cannot combine both xml and json output"); + boolean jsonl = line.hasOption("jsonl"); + boolean xmlOutput = line.hasOption("xmloutput"); + if ((jsonOutput || jsonl) && xmlOutput) { + throw new IllegalArgumentException("Cannot combine both XML and JSON output"); + } else if (jsonOutput && jsonl) { + throw new IllegalArgumentException("Cannot combine both JSON and JSONL output"); + } + if (jsonl) { + allParams.setJsonLinesOutput(true); + } else { + allParams.setJsonOutput(!xmlOutput); + } + + if (allParams.slices() != 1 || allParams.sliceId() != 0) { + if ((allParams.slices() < 1) || (allParams.sliceId() < 0) || (allParams.sliceId() >= allParams.slices())) { + throw new IllegalArgumentException("--slices must be greater than 0 and --sliceid must be in the " + + "range [0, the value provided for --slices)"); + } + params.slice(allParams.slices(), allParams.sliceId()); } - allParams.setJsonOutput(!xmlOutput); allParams.setVisitorParameters(params); return allParams; @@ -691,6 +807,9 @@ public class VdsVisit { if (params.skipBucketsOnFatalErrors()) { out.println("Skip visiting super buckets with fatal errors."); } + if (params.getSlices() > 1) { + out.format("Visiting slice %d out of %s slices\n", params.getSliceId(), params.getSlices()); + } } private void onDocumentSelectionException(Exception e) { @@ -716,24 +835,14 @@ public class VdsVisit { !"".equals(visitorParameters.getResumeFileName())) { try { - File file = new File(visitorParameters.getResumeFileName()); - FileInputStream fos = new FileInputStream(file); - - StringBuilder builder = new StringBuilder(); - byte[] b = new byte[100000]; - int length; - - while ((length = fos.read(b)) > 0) { - builder.append(new String(b, 0, length)); - } - fos.close(); - visitorParameters.setResumeToken(new ProgressToken(builder.toString())); + var progressFileContents = Files.readString(Path.of(visitorParameters.getResumeFileName())); + visitorParameters.setResumeToken(new ProgressToken(progressFileContents)); if (params.isVerbose()) { System.err.format("Resuming visitor already %.1f %% finished.\n", visitorParameters.getResumeToken().percentFinished()); } - } catch (FileNotFoundException e) { + } catch (NoSuchFileException e) { // Ignore; file has not been created yet but will be shortly. } catch (IOException e) { System.err.println("Could not open progress file: " + visitorParameters.getResumeFileName()); @@ -747,17 +856,19 @@ public class VdsVisit { VdsVisitHandler handler; - handler = new StdOutVisitorHandler( - params.isPrintIdsOnly(), - params.isVerbose(), - params.isVerbose(), - params.isVerbose(), - params.getStatisticsParts() != null, - params.getAbortOnClusterDown(), - params.getProcessTime(), - params.jsonOutput, - params.tensorShortForm, - params.tensorDirectValues); + var handlerParams = new StdOutVisitorHandler.Params(); + handlerParams.printIds = params.isPrintIdsOnly(); + handlerParams.indentXml = params.isVerbose(); + handlerParams.showProgress = params.isVerbose(); + handlerParams.showStatistics = params.isVerbose(); + handlerParams.doStatistics = params.getStatisticsParts() != null; + handlerParams.abortOnClusterDown = params.getAbortOnClusterDown(); + handlerParams.processTimeMilliSecs = params.getProcessTime(); + handlerParams.outputFormat = params.stdOutHandlerOutputFormat(); + handlerParams.tensorShortForm = params.tensorShortForm(); + handlerParams.tensorDirectValues = params.tensorDirectValues(); + handlerParams.nullRender = params.nullRender(); + handler = new StdOutVisitorHandler(handlerParams); if (visitorParameters.getResumeFileName() != null) { handler.setProgressFileName(visitorParameters.getResumeFileName()); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java index ea861399e76..ccb0888a654 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java @@ -6,16 +6,20 @@ import com.yahoo.documentapi.VisitorControlHandler; import com.yahoo.documentapi.VisitorDataHandler; import com.yahoo.vdslib.VisitorStatistics; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; import java.util.Date; import java.util.TimeZone; import java.text.DateFormat; import java.text.DecimalFormat; import java.text.SimpleDateFormat; +import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + /** * An abstract class that can be subclassed by different visitor handlers. * @@ -30,14 +34,26 @@ public abstract class VdsVisitHandler { String lastPercentage; final Object printLock = new Object(); - protected String progressFileName = ""; + private static class ProgressMeta { + String fileName = ""; + String lastProgressContents; + int unwrittenUpdates = 0; + long lastWriteAtNanos = 0; + Duration writeInterval = Duration.ofSeconds(10); + boolean shouldWriteProgress() { + return !fileName.isEmpty(); + } + } + + ProgressMeta progressMeta = new ProgressMeta(); final VisitorControlHandler controlHandler = new ControlHandler(); public VdsVisitHandler(boolean showProgress, boolean showStatistics, boolean abortOnClusterDown) { this.showProgress = showProgress; this.showStatistics = showStatistics; this.abortOnClusterDown = abortOnClusterDown; + this.progressMeta.lastWriteAtNanos = System.nanoTime(); // Avoid always writing a file on the first progress update } public boolean getShowProgress() { @@ -75,11 +91,11 @@ public abstract class VdsVisitHandler { public void onDone() { } public String getProgressFileName() { - return progressFileName; + return progressMeta.fileName; } public void setProgressFileName(String progressFileName) { - this.progressFileName = progressFileName; + this.progressMeta.fileName = progressFileName; } public VisitorControlHandler getControlHandler() { return controlHandler; } @@ -88,20 +104,29 @@ public abstract class VdsVisitHandler { class ControlHandler extends VisitorControlHandler { VisitorStatistics statistics; + private void rewriteProgressFile() { + try { + var tmpPath = Path.of(progressMeta.fileName + ".tmp"); + Files.writeString(tmpPath, progressMeta.lastProgressContents); + Files.move(tmpPath, Path.of(progressMeta.fileName), REPLACE_EXISTING, ATOMIC_MOVE); + } catch (IOException e) { + e.printStackTrace(); + abort(); // Don't continue visiting if we're unable to save progress state + } + } + public void onProgress(ProgressToken token) { - if (progressFileName.length() > 0) { - try { - synchronized (token) { - File file = new File(progressFileName + ".tmp"); - FileOutputStream fos = new FileOutputStream(file); - fos.write(token.toString().getBytes()); - fos.close(); - file.renameTo(new File(progressFileName)); + if (progressMeta.shouldWriteProgress()) { + synchronized (token) { + progressMeta.unwrittenUpdates++; + progressMeta.lastProgressContents = token.toString(); + long nowNanos = System.nanoTime(); + if ((nowNanos - progressMeta.lastWriteAtNanos) > progressMeta.writeInterval.toNanos()) { + rewriteProgressFile(); + progressMeta.unwrittenUpdates = 0; + progressMeta.lastWriteAtNanos = nowNanos; } } - catch (IOException e) { - e.printStackTrace(); - } } if (showProgress) { synchronized (printLock) { @@ -111,7 +136,9 @@ public abstract class VdsVisitHandler { if (lastLineIsProgress) { System.err.print('\r'); } - System.err.print(percentage + " % finished."); + // Pad with a few extra spaces to handle case where current line written is shorter + // than the previous line written. Would otherwise leave stale characters behind. + System.err.print(percentage + " % finished. "); lastLineIsProgress = true; lastPercentage = percentage; } @@ -147,6 +174,11 @@ public abstract class VdsVisitHandler { } } public void onDone(CompletionCode code, String message) { + // Flush any remaining unwritten progress updates. + // It is expected that this happens-after any and all calls to onProgress(). + if (progressMeta.unwrittenUpdates > 0) { + rewriteProgressFile(); + } if (lastLineIsProgress) { System.err.print('\n'); lastLineIsProgress = false; diff --git a/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java b/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java index c1bbe8711a5..1ebbc8ac6ad 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java @@ -1,15 +1,19 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespavisit; +import com.yahoo.document.DataType; import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentType; import com.yahoo.document.TensorDataType; +import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.documentapi.AckToken; import com.yahoo.documentapi.VisitorControlSession; import com.yahoo.documentapi.VisitorDataHandler; import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import org.junit.jupiter.api.Test; @@ -26,23 +30,27 @@ import static org.mockito.Mockito.mock; * @author bjorncs */ public class StdOutVisitorHandlerTest { - private boolean jsonOutput; - - public void initStdOutVisitorHandlerTest(boolean jsonOutput) { - this.jsonOutput = jsonOutput; - } public static Object[] data() { return new Object[]{true, false}; } + private static StdOutVisitorHandler.Params createHandlerParams(boolean jsonOutput, boolean tensorShortForm, boolean tensorDirectValues) { + var params = new StdOutVisitorHandler.Params(); + params.outputFormat = jsonOutput ? StdOutVisitorHandler.OutputFormat.JSON + : StdOutVisitorHandler.OutputFormat.XML; + params.tensorShortForm = tensorShortForm; + params.tensorDirectValues = tensorDirectValues; + return params; + } + @MethodSource("data") @ParameterizedTest(name = "jsonOutput={0}") void printing_ids_for_zero_documents_produces_empty_output(boolean jsonOutput) { - initStdOutVisitorHandlerTest(jsonOutput); ByteArrayOutputStream out = new ByteArrayOutputStream(); - StdOutVisitorHandler visitorHandler = - new StdOutVisitorHandler(/*printIds*/true, false, false, false, false, false, 0, jsonOutput, false, false, new PrintStream(out, true)); + var params = createHandlerParams(jsonOutput, false, false); + params.printIds = true; + StdOutVisitorHandler visitorHandler = new StdOutVisitorHandler(params, new PrintStream(out, true)); VisitorDataHandler dataHandler = visitorHandler.getDataHandler(); dataHandler.onDone(); String output = out.toString(); @@ -52,10 +60,9 @@ public class StdOutVisitorHandlerTest { @MethodSource("data") @ParameterizedTest(name = "jsonOutput={0}") void printing_zero_documents_produces_empty_output(boolean jsonOutput) { - initStdOutVisitorHandlerTest(jsonOutput); ByteArrayOutputStream out = new ByteArrayOutputStream(); StdOutVisitorHandler visitorHandler = - new StdOutVisitorHandler(/*printIds*/false, false, false, false, false, false, 0, jsonOutput, false, false, new PrintStream(out, true)); + new StdOutVisitorHandler(createHandlerParams(jsonOutput, false, false), new PrintStream(out, true)); VisitorDataHandler dataHandler = visitorHandler.getDataHandler(); dataHandler.onDone(); String expectedOutput = jsonOutput ? "[]" : ""; @@ -71,8 +78,7 @@ public class StdOutVisitorHandlerTest { var putMsg = new PutDocumentMessage(new DocumentPut(doc)); var out = new ByteArrayOutputStream(); - var visitorHandler = new StdOutVisitorHandler(/*printIds*/false, false, false, false, false, false, - 0, true, tensorShortForm, tensorDirectValues, new PrintStream(out, true)); + var visitorHandler = new StdOutVisitorHandler(createHandlerParams(true, tensorShortForm, tensorDirectValues), new PrintStream(out, true)); var dataHandler = visitorHandler.getDataHandler(); var controlSession = mock(VisitorControlSession.class); var ackToken = mock(AckToken.class); @@ -100,4 +106,77 @@ public class StdOutVisitorHandlerTest { do_test_json_tensor_fields_rendering(true, false, expectedOutput); } + private static PutDocumentMessage createPutWithDocAndValue(DocumentType docType, String docId, String fieldValue) { + var doc = new Document(docType, docId); + doc.setFieldValue("bar", new StringFieldValue(fieldValue)); + return new PutDocumentMessage(new DocumentPut(doc)); + } + + private static RemoveDocumentMessage createRemoveForDoc(String docId) { + return new RemoveDocumentMessage(new DocumentId(docId)); + } + + @MethodSource("data") + @ParameterizedTest(name = "jsonLinesFormat={0}") + void json_can_be_output_in_json_lines_format(boolean jsonLinesFormat) { + var docType = new DocumentType("foo"); + docType.addField("bar", DataType.STRING); + + var params = createHandlerParams(true, true, true); + params.outputFormat = jsonLinesFormat ? StdOutVisitorHandler.OutputFormat.JSONL + : StdOutVisitorHandler.OutputFormat.JSON; + + var out = new ByteArrayOutputStream(); + var visitorHandler = new StdOutVisitorHandler(params, new PrintStream(out, true)); + var dataHandler = visitorHandler.getDataHandler(); + var controlSession = mock(VisitorControlSession.class); + dataHandler.setSession(controlSession); + + dataHandler.onMessage(createPutWithDocAndValue(docType, "id:baz:foo::1", "fluffy\nbunnies"), mock(AckToken.class)); + dataHandler.onMessage(createRemoveForDoc("id:baz:foo::2"), mock(AckToken.class)); + dataHandler.onMessage(createPutWithDocAndValue(docType, "id:baz:foo::3", "\r\ncool fox\r\n"), mock(AckToken.class)); + dataHandler.onDone(); + + String output = out.toString().trim(); + if (jsonLinesFormat) { + // JSONL; no implicit start/end array chars or trailing commas after objects + var expected = """ + {"id":"id:baz:foo::1","fields":{"bar":"fluffy\\nbunnies"}} + {"remove":"id:baz:foo::2"} + {"id":"id:baz:foo::3","fields":{"bar":"\\r\\ncool fox\\r\\n"}}"""; + assertEquals(expected, output); + } else { + // non-JSONL; usual array of comma-separated objects form + var expected = """ + [ + {"id":"id:baz:foo::1","fields":{"bar":"fluffy\\nbunnies"}}, + {"remove":"id:baz:foo::2"}, + {"id":"id:baz:foo::3","fields":{"bar":"\\r\\ncool fox\\r\\n"}}]"""; + assertEquals(expected, output); + } + } + + @Test + void nothing_is_rendered_if_null_render_option_is_specified() { + var docType = new DocumentType("foo"); + docType.addField("bar", DataType.STRING); + + var params = createHandlerParams(true, true, true); + params.nullRender = true; + + var out = new ByteArrayOutputStream(); + var visitorHandler = new StdOutVisitorHandler(params, new PrintStream(out, true)); + var dataHandler = visitorHandler.getDataHandler(); + var controlSession = mock(VisitorControlSession.class); + dataHandler.setSession(controlSession); + + dataHandler.onMessage(createPutWithDocAndValue(docType, "id:baz:foo::1", "fluffy\nbunnies"), mock(AckToken.class)); + dataHandler.onMessage(createRemoveForDoc("id:baz:foo::2"), mock(AckToken.class)); + dataHandler.onMessage(createPutWithDocAndValue(docType, "id:baz:foo::3", "\r\ncool fox\r\n"), mock(AckToken.class)); + dataHandler.onDone(); + + String output = out.toString().trim(); + assertEquals("", output); + } + } diff --git a/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java b/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java index 434d91b3ea3..a0db3f973a4 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java @@ -19,8 +19,11 @@ import org.junit.jupiter.api.Test; import java.io.ByteArrayOutputStream; import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -77,7 +80,6 @@ public class VdsVisitTestCase { /** * Test the parameters that could not be used in conjunction with * those in the first parameter test. - * @throws Exception */ @Test void testCommandLineShortOptions2() throws Exception { @@ -113,6 +115,11 @@ public class VdsVisitTestCase { assertTrue(allParams.isPrintIdsOnly()); } + private static String joinLines(String... lines) { + String nl = System.getProperty("line.separator"); // in case of \r\n instead of just \n... + return String.join(nl, lines) + nl; + } + @Test void testCommandLineLongOptions() throws Exception { // short options testing (for options that do not collide with each other) @@ -140,7 +147,9 @@ public class VdsVisitTestCase { "--abortonclusterdown", "--visitremoves", "--bucketspace", "outerspace", - "--shorttensors" + "--shorttensors", + "--slices", "16", + "--sliceid", "5" }; VdsVisit.ArgumentParser parser = createMockArgumentParser(); VdsVisit.VdsVisitParameters allParams = parser.parse(args); @@ -177,33 +186,44 @@ public class VdsVisitTestCase { assertTrue(allParams.getAbortOnClusterDown()); assertTrue(params.visitRemoves()); + assertEquals(16, params.getSlices()); + assertEquals(5, params.getSliceId()); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); PrintStream printStream = new PrintStream(outputStream); VdsVisit.verbosePrintParameters(allParams, printStream); printStream.flush(); String nl = System.getProperty("line.separator"); // the joys of running tests on windows - assertEquals( - "Time out visitor after 123456789 ms." + nl + - "Visiting documents matching: 'id.user=1234'" + nl + - "Visiting bucket space: outerspace" + nl + - "Visiting in the inclusive timestamp range 5678 - 9012." + nl + - "Visiting field set foodoc.bar,foodoc.baz." + nl + - "Visiting inconsistent buckets." + nl + - "Including remove entries." + nl + - "Tracking progress in file: foo-progress.txt" + nl + - "Let visitor have maximum 6000 replies pending on data handlers per storage node visitor." + nl + - "Visit maximum 5 buckets per visitor." + nl + - "Sending data to data handler at: foo.remote" + nl + - "Using visitor library 'fnord'." + nl + - "Adding the following library specific parameters:" + nl + - " asdf = rargh" + nl + - "Visitor priority NORMAL_1" + nl + - "Skip visiting super buckets with fatal errors." + nl, - outputStream.toString("utf-8")); + assertEquals(joinLines( + "Time out visitor after 123456789 ms.", + "Visiting documents matching: 'id.user=1234'", + "Visiting bucket space: outerspace", + "Visiting in the inclusive timestamp range 5678 - 9012.", + "Visiting field set foodoc.bar,foodoc.baz.", + "Visiting inconsistent buckets.", + "Including remove entries.", + "Tracking progress in file: foo-progress.txt", + "Let visitor have maximum 6000 replies pending on data handlers per storage node visitor.", + "Visit maximum 5 buckets per visitor.", + "Sending data to data handler at: foo.remote", + "Using visitor library 'fnord'.", + "Adding the following library specific parameters:", + " asdf = rargh", + "Visitor priority NORMAL_1", + "Skip visiting super buckets with fatal errors.", + "Visiting slice 5 out of 16 slices"), + outputStream.toString(StandardCharsets.UTF_8)); } private static String[] emptyArgList() { return new String[]{}; } + @Test + void slicing_is_disabled_by_default() throws Exception { + var allParams = createMockArgumentParser().parse(emptyArgList()); + assertEquals(1, allParams.slices()); // 1 slice; the entire cluster + assertEquals(0, allParams.sliceId()); + } + // TODO Vespa 9: change default from long to short @Test void tensor_output_format_is_long_by_default() throws Exception { diff --git a/vespaclient/CMakeLists.txt b/vespaclient/CMakeLists.txt index 912b35fa763..6e82b83517e 100644 --- a/vespaclient/CMakeLists.txt +++ b/vespaclient/CMakeLists.txt @@ -2,7 +2,6 @@ vespa_define_module( DEPENDS vespadefaults - fastos configdefinitions config_cloudconfig vespalog diff --git a/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp b/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp index f70ffcc2655..50311e772e2 100644 --- a/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp +++ b/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp @@ -501,8 +501,8 @@ Application::isService(FRT_Supervisor &frt, const std::string &spec) const } } - req->SubRef(); - target->SubRef(); + req->internal_subref(); + target->internal_subref(); return ret; } diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/ThreadLocalDirectory.java b/vespajlib/src/main/java/com/yahoo/concurrent/ThreadLocalDirectory.java index 15ba65310ff..c28884696aa 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/ThreadLocalDirectory.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/ThreadLocalDirectory.java @@ -241,8 +241,7 @@ public final class ThreadLocalDirectory<AGGREGATOR, SAMPLE> { synchronized (directoryLock) { previousIntervalSize = directory.size(); previous = directory; - directory = new ArrayList<>( - previousIntervalSize); + directory = new ArrayList<>(previousIntervalSize); } contained = new ArrayList<>(previousIntervalSize); // Yes, this is an inconsistence about when the registered state is @@ -268,12 +267,10 @@ public final class ThreadLocalDirectory<AGGREGATOR, SAMPLE> { throw new IllegalStateException("Does not use observable updaters."); } List<LocalInstance<AGGREGATOR, SAMPLE>> current; - List<AGGREGATOR> view; synchronized (directoryLock) { - current = new ArrayList<>( - directory); + current = new ArrayList<>(directory); } - view = new ArrayList<>(current.size()); + List<AGGREGATOR> view = new ArrayList<>(current.size()); for (LocalInstance<AGGREGATOR, SAMPLE> x : current) { view.add(x.copyCurrent(observableUpdater)); } diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java index 1e2c0900ff7..33e46ebc75f 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java @@ -93,7 +93,7 @@ public abstract class Maintainer implements Runnable { * * @return the degree to which the run was successful - a number between 0 (no success), to 1 (complete success). * Note that this indicates whether something is wrong, so e.g if the call did nothing because it should do - * nothing, 1.0 should be returned. + * nothing, 1.0 should be returned. */ protected abstract double maintain(); diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 2720d8786cb..dc99146fd3f 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -6,7 +6,6 @@ endif() vespa_define_module( DEPENDS - fastos vespalog EXTERNAL_DEPENDS @@ -26,6 +25,8 @@ vespa_define_module( src/apps/vespa-validate-hostname TESTS + ${VESPALIB_DIRECTIO_TESTDIR} + ${VESPALIB_PROCESS_MEMORY_STATS_TESTDIR} src/tests/alloc src/tests/approx src/tests/array @@ -38,9 +39,9 @@ vespa_define_module( src/tests/bits src/tests/box src/tests/btree - src/tests/btree/btree_store src/tests/btree/btree-scan-speed src/tests/btree/btree-stress + src/tests/btree/btree_store src/tests/clock src/tests/component src/tests/compress @@ -73,7 +74,6 @@ vespa_define_module( src/tests/datastore/unique_store src/tests/datastore/unique_store_dictionary src/tests/datastore/unique_store_string_allocator - ${VESPALIB_DIRECTIO_TESTDIR} src/tests/detect_type_benchmark src/tests/dotproduct src/tests/drop-file-from-cache @@ -86,6 +86,9 @@ vespa_define_module( src/tests/executor_idle_tracking src/tests/explore_modern_cpp src/tests/false + src/tests/fastlib/io + src/tests/fastlib/text + src/tests/fastos src/tests/fiddle src/tests/fileheader src/tests/floatingpointtype @@ -95,6 +98,7 @@ vespa_define_module( src/tests/guard src/tests/host_name src/tests/hwaccelrated + src/tests/invokeservice src/tests/io/fileutil src/tests/io/mapped_file_input src/tests/issue @@ -134,24 +138,23 @@ vespa_define_module( src/tests/printable src/tests/priority_queue src/tests/process - ${VESPALIB_PROCESS_MEMORY_STATS_TESTDIR} src/tests/programoptions src/tests/random - src/tests/referencecounter + src/tests/ref_counted src/tests/regex src/tests/rendezvous src/tests/require src/tests/runnable_pair src/tests/rusage src/tests/sequencedtaskexecutor - src/tests/shutdownguard - src/tests/singleexecutor src/tests/sha1 src/tests/shared_operation_throttler src/tests/shared_string_repo src/tests/sharedptr + src/tests/shutdownguard src/tests/signalhandler src/tests/simple_thread_bundle + src/tests/singleexecutor src/tests/slime src/tests/slime/external_data_value src/tests/slime/summary-feature-benchmark @@ -204,17 +207,15 @@ vespa_define_module( src/tests/util/string_escape src/tests/valgrind src/tests/visit_ranges - src/tests/invokeservice src/tests/wakeup src/tests/xmlserializable src/tests/zcurve - src/tests/fastlib/io - src/tests/fastlib/text LIBS src/vespa/fastlib/io src/vespa/fastlib/text src/vespa/fastlib/text/apps + src/vespa/fastos src/vespa/vespalib src/vespa/vespalib/btree src/vespa/vespalib/component diff --git a/vespalib/src/tests/btree/iteratespeed.cpp b/vespalib/src/tests/btree/iteratespeed.cpp index fceaf01a785..48c4b4a1c39 100644 --- a/vespalib/src/tests/btree/iteratespeed.cpp +++ b/vespalib/src/tests/btree/iteratespeed.cpp @@ -1,10 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/btree/btreeroot.h> #include <vespa/vespalib/btree/btreebuilder.h> #include <vespa/vespalib/btree/btreenodeallocator.h> #include <vespa/vespalib/btree/btree.h> -#include <vespa/vespalib/btree/btreestore.h> #include <vespa/vespalib/btree/btreenodeallocator.hpp> #include <vespa/vespalib/btree/btreenode.hpp> #include <vespa/vespalib/btree/btreenodestore.hpp> @@ -12,11 +10,10 @@ #include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/vespalib/btree/btreebuilder.hpp> #include <vespa/vespalib/btree/btree.hpp> -#include <vespa/vespalib/btree/btreestore.hpp> #include <vespa/vespalib/datastore/buffer_type.hpp> #include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/util/time.h> - +#include <cinttypes> #include <unistd.h> #include <vespa/log/log.h> diff --git a/vespalib/src/tests/clock/clock_benchmark.cpp b/vespalib/src/tests/clock/clock_benchmark.cpp index a21ad2b05ef..620b4e5c83c 100644 --- a/vespalib/src/tests/clock/clock_benchmark.cpp +++ b/vespalib/src/tests/clock/clock_benchmark.cpp @@ -2,7 +2,6 @@ #include <vespa/vespalib/util/clock.h> #include <vespa/vespalib/util/invokeserviceimpl.h> -#include <vespa/fastos/thread.h> #include <cassert> #include <vector> #include <atomic> @@ -10,6 +9,7 @@ #include <cstring> #include <condition_variable> #include <mutex> +#include <thread> using vespalib::Clock; using vespalib::steady_time; @@ -36,7 +36,7 @@ struct NSAtomic : public UpdateClock { std::atomic<int64_t> _value; }; -class TestClock : public FastOS_Runnable +class TestClock { private: int _timePeriodMS; @@ -44,8 +44,9 @@ private: std::condition_variable _cond; UpdateClock &_clock; bool _stop; + std::thread _thread; - void Run(FastOS_ThreadInterface *thisThread, void *arguments) override; + void run(); public: TestClock(UpdateClock & clock, double timePeriod) @@ -53,74 +54,68 @@ public: _lock(), _cond(), _clock(clock), - _stop(false) - { } + _stop(false), + _thread() + { + _thread = std::thread([this](){run();}); + } ~TestClock() { - std::lock_guard<std::mutex> guard(_lock); - _stop = true; - _cond.notify_all(); + { + std::lock_guard<std::mutex> guard(_lock); + _stop = true; + _cond.notify_all(); + } + _thread.join(); } }; -void TestClock::Run(FastOS_ThreadInterface *thread, void *) +void TestClock::run() { std::unique_lock<std::mutex> guard(_lock); - while ( ! thread->GetBreakFlag() && !_stop) { + while (!_stop) { _clock.update(); _cond.wait_for(guard, std::chrono::milliseconds(_timePeriodMS)); } } -struct SamplerBase : public FastOS_Runnable { - SamplerBase(uint32_t threadId) - : _thread(nullptr), - _threadId(threadId), - _samples(0), - _count() +template<typename Func> +struct Sampler { + Sampler(Func func, uint64_t samples) + : _samples(samples), + _count(), + _func(func), + _thread() { memset(_count, 0, sizeof(_count)); + _thread = std::thread([this](){run();}); } - FastOS_ThreadInterface * _thread; - uint32_t _threadId; - uint64_t _samples; - uint64_t _count[3]; -}; - -template<typename Func> -struct Sampler : public SamplerBase { - Sampler(Func func, uint32_t threadId) - : SamplerBase(threadId), - _func(func) - { } - void Run(FastOS_ThreadInterface *, void *) override { - uint64_t samples; + void run() { steady_time prev = _func(); - for (samples = 0; (samples < _samples); samples++) { + for (uint64_t samples = 0; samples < _samples; ++samples) { steady_time now = _func(); duration diff = now - prev; if (diff > duration::zero()) prev = now; _count[1 + ((diff == duration::zero()) ? 0 : (diff > duration::zero()) ? 1 : -1)]++; } - } - Func _func; + uint64_t _samples; + uint64_t _count[3]; + Func _func; + std::thread _thread; }; template<typename Func> -void benchmark(const char * desc, FastOS_ThreadPool & pool, uint64_t samples, uint32_t numThreads, Func func) { - std::vector<std::unique_ptr<SamplerBase>> threads; +void benchmark(const char * desc, uint64_t samples, uint32_t numThreads, Func func) { + std::vector<std::unique_ptr<Sampler<Func>>> threads; threads.reserve(numThreads); steady_time start = steady_clock::now(); for (uint32_t i(0); i < numThreads; i++) { - SamplerBase * sampler = new Sampler<Func>(func, i); - sampler->_samples = samples; - sampler->_thread = pool.NewThread(sampler, nullptr); - threads.emplace_back(sampler); + threads.push_back(std::make_unique<Sampler<Func>>(func, samples)); } uint64_t count[3]; memset(count, 0, sizeof(count)); for (const auto & sampler : threads) { - sampler->_thread->Join(); + sampler->_thread.join(); for (uint32_t i(0); i < 3; i++) { count[i] += sampler->_count[i]; } @@ -129,12 +124,15 @@ void benchmark(const char * desc, FastOS_ThreadPool & pool, uint64_t samples, ui } int -main(int , char *argv[]) +main(int argc, char *argv[]) { + if (argc != 4) { + fprintf(stderr, "usage: %s <frequency> <numThreads> <samples>\n", argv[0]); + return 1; + } uint64_t frequency = atoll(argv[1]); uint32_t numThreads = atoi(argv[2]); uint64_t samples = atoll(argv[3]); - FastOS_ThreadPool pool; NSValue nsValue; NSVolatile nsVolatile; NSAtomic nsAtomic; @@ -143,36 +141,30 @@ main(int , char *argv[]) TestClock nsClock(nsValue, 1.0/frequency); TestClock nsVolatileClock(nsVolatile, 1.0/frequency); TestClock nsAtomicClock(nsAtomic, 1.0/frequency); - assert(pool.NewThread(&nsClock, nullptr) != nullptr); - assert(pool.NewThread(&nsVolatileClock, nullptr) != nullptr); - assert(pool.NewThread(&nsAtomicClock, nullptr) != nullptr); - benchmark("vespalib::Clock", pool, samples, numThreads, [&clock]() { + benchmark("vespalib::Clock", samples, numThreads, [&clock]() { return clock.getTimeNS(); }); - benchmark("uint64_t", pool, samples, numThreads, [&nsValue]() { + benchmark("uint64_t", samples, numThreads, [&nsValue]() { return steady_time (duration(nsValue._value)); }); - benchmark("volatile uint64_t", pool, samples, numThreads, [&nsVolatile]() { + benchmark("volatile uint64_t", samples, numThreads, [&nsVolatile]() { return steady_time(duration(nsVolatile._value)); }); - benchmark("memory_order_relaxed", pool, samples, numThreads, [&nsAtomic]() { + benchmark("memory_order_relaxed", samples, numThreads, [&nsAtomic]() { return steady_time(duration(nsAtomic._value.load(std::memory_order_relaxed))); }); - benchmark("memory_order_consume", pool, samples, numThreads, [&nsAtomic]() { + benchmark("memory_order_consume", samples, numThreads, [&nsAtomic]() { return steady_time(duration(nsAtomic._value.load(std::memory_order_consume))); }); - benchmark("memory_order_acquire", pool, samples, numThreads, [&nsAtomic]() { + benchmark("memory_order_acquire", samples, numThreads, [&nsAtomic]() { return steady_time(duration(nsAtomic._value.load(std::memory_order_acquire))); }); - benchmark("memory_order_seq_cst", pool, samples, numThreads, [&nsAtomic]() { + benchmark("memory_order_seq_cst", samples, numThreads, [&nsAtomic]() { return steady_time(duration(nsAtomic._value.load(std::memory_order_seq_cst))); }); - - benchmark("vespalib::steady_time::now()", pool, samples, numThreads, []() { + benchmark("vespalib::steady_time::now()", samples, numThreads, []() { return steady_clock::now(); }); - - pool.Close(); return 0; } diff --git a/vespalib/src/tests/datastore/datastore/datastore_test.cpp b/vespalib/src/tests/datastore/datastore/datastore_test.cpp index 645871d3ef6..e17ac94775e 100644 --- a/vespalib/src/tests/datastore/datastore/datastore_test.cpp +++ b/vespalib/src/tests/datastore/datastore/datastore_test.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/test/memory_allocator_observer.h> #include <vespa/vespalib/util/size_literals.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP("datastore_test"); diff --git a/vespalib/src/tests/fastos/CMakeLists.txt b/vespalib/src/tests/fastos/CMakeLists.txt new file mode 100644 index 00000000000..6ea986ac0b0 --- /dev/null +++ b/vespalib/src/tests/fastos/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(fastos_file_test_app TEST + SOURCES + file_test.cpp + DEPENDS + vespalib + GTest::GTest +) +vespa_add_test(NAME fastos_file_test_app COMMAND fastos_file_test_app) diff --git a/vespalib/src/tests/fastos/file_test.cpp b/vespalib/src/tests/fastos/file_test.cpp new file mode 100644 index 00000000000..6b58a4a1fd8 --- /dev/null +++ b/vespalib/src/tests/fastos/file_test.cpp @@ -0,0 +1,233 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/file.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <memory> +#include <cassert> +#include <sys/mman.h> +#include <filesystem> + +const std::string srcDir = getenv("SOURCE_DIRECTORY") ? getenv("SOURCE_DIRECTORY") : "."; +const std::string roFilename = srcDir + "/hello.txt"; +const std::string woFilename = "generated/writeonlytest.txt"; +const std::string rwFilename = "generated/readwritetest.txt"; + +// create and remove 'generated' sub-directory +struct Generated { + Generated() { std::filesystem::create_directory(std::filesystem::path("generated")); } + ~Generated() { std::filesystem::remove_all(std::filesystem::path("generated")); } +}; + +TEST(FileTest, GetCurrentDirTest) { + std::string currentDir = FastOS_File::getCurrentDirectory(); + EXPECT_FALSE(currentDir.empty()); + EXPECT_TRUE(FastOS_File::SetCurrentDirectory("..")); + std::string parentDir = FastOS_File::getCurrentDirectory(); + EXPECT_FALSE(parentDir.empty()); + EXPECT_NE(currentDir, parentDir); + EXPECT_TRUE(FastOS_File::SetCurrentDirectory(currentDir.c_str())); +} + +void MemoryMapTestImpl(int mmap_flags) { + Generated guard; + const int bufSize = 1000; + FastOS_File file("generated/memorymaptest"); + ASSERT_TRUE(file.OpenReadWrite()); + std::vector<char> space(bufSize); + char *buffer = space.data(); + for (int i = 0; i < bufSize; i++) { + buffer[i] = i % 256; + } + EXPECT_EQ(file.Write2(buffer, bufSize), bufSize); + bool close_ok = file.Close(); + assert(close_ok); + file.enableMemoryMap(mmap_flags); + ASSERT_TRUE(file.OpenReadOnly()); + bool mmapEnabled = file.IsMemoryMapped(); + char *mmapBuffer = static_cast<char *>(file.MemoryMapPtr(0)); + fprintf(stderr, "Memory mapping %s\n", mmapEnabled ? "enabled" : "disabled"); + fprintf(stderr, "Map address: 0x%p\n", mmapBuffer); + if (mmapEnabled) { + for (int i = 0; i < bufSize; i++) { + EXPECT_EQ(mmapBuffer[i], char(i % 256)); + } + } +} + +TEST(FileTest, MemoryMapTest) { MemoryMapTestImpl(0); } + +#ifdef __linux__ +TEST(FileTest, MemoryMapTestHuge) { MemoryMapTestImpl(MAP_HUGETLB); } +#endif + +TEST(FileTest, DirectIOTest) { + Generated guard; + const int bufSize = 40000; + FastOS_File file("generated/diotest"); + ASSERT_TRUE(file.OpenWriteOnly()); + std::vector<char> space(bufSize); + char *buffer = space.data(); + for (int i = 0; i < bufSize; i++) { + buffer[i] = 'A' + (i % 17); + } + EXPECT_EQ(file.Write2(buffer, bufSize), bufSize); + bool close_ok = file.Close(); + assert(close_ok); + file.EnableDirectIO(); + ASSERT_TRUE(file.OpenReadOnly()); + size_t memoryAlignment = 0; + size_t transferGranularity = 0; + size_t transferMaximum = 0; + bool dioEnabled = file.GetDirectIORestrictions(memoryAlignment, + transferGranularity, + transferMaximum); + fprintf(stderr, "DirectIO %s\n", dioEnabled ? "enabled" : "disabled"); + fprintf(stderr, "Memory alignment: %zu bytes\n", memoryAlignment); + fprintf(stderr, "Transfer granularity: %zu bytes\n", transferGranularity); + fprintf(stderr, "Transfer maximum: %zu bytes\n", transferMaximum); + if (dioEnabled) { + int eachRead = (8192 + transferGranularity - 1) / transferGranularity; + std::vector<char> space2(eachRead * transferGranularity + memoryAlignment - 1); + char *buffer2 = space2.data(); + char *alignPtr = buffer2; + unsigned int align = + static_cast<unsigned int> + (reinterpret_cast<unsigned long>(alignPtr) & + (memoryAlignment - 1)); + if (align != 0) { + alignPtr = &alignPtr[memoryAlignment - align]; + } + int residue = bufSize; + int pos = 0; + while (residue > 0) { + int readThisTime = eachRead * transferGranularity; + if (readThisTime > residue) { + readThisTime = residue; + } + file.ReadBuf(alignPtr, readThisTime, pos); + for (int i = 0; i < readThisTime; i++) { + ASSERT_EQ(alignPtr[i], char('A' + ((i+pos) % 17))); + } + residue -= readThisTime; + pos += readThisTime; + } + ASSERT_TRUE(file.SetPosition(1)); + try { + const int attemptReadBytes = 173; + [[maybe_unused]] auto res = file.Read(buffer, attemptReadBytes); + EXPECT_TRUE(false); + } catch (const DirectIOException &) { + fprintf(stderr, "got DirectIOException as expected\n"); + } catch (...) { + EXPECT_TRUE(false); + } + ASSERT_TRUE(file.SetPosition(1)); + try { + const int attemptReadBytes = 4096; + [[maybe_unused]] auto res = file.Read(buffer, attemptReadBytes); + EXPECT_TRUE(false); + } catch (const DirectIOException &) { + fprintf(stderr, "got DirectIOException as expected\n"); + } catch (...) { + EXPECT_TRUE(false); + } + } else { + memset(buffer, 0, bufSize); + ssize_t readBytes = file.Read(buffer, bufSize); + ASSERT_EQ(readBytes, bufSize); + for (int i = 0; i < bufSize; i++) { + ASSERT_EQ(buffer[i], char('A' + (i % 17))); + } + } +} + +TEST(FileTest, ReadOnlyTest) { + auto myFile = std::make_unique<FastOS_File>(roFilename.c_str()); + ASSERT_TRUE(myFile->OpenReadOnly()); + EXPECT_EQ(myFile->GetSize(), 27); + char dummyData[6] = "Dummy"; + ASSERT_FALSE(myFile->CheckedWrite(dummyData, 6)); + char dummyData2[28]; + ASSERT_TRUE(myFile->SetPosition(1)); + EXPECT_EQ(myFile->Read(dummyData2, 28), 26); + EXPECT_EQ(myFile->GetPosition(), 27); +} + +TEST(FileTest, WriteOnlyTest) { + Generated guard; + auto myFile = std::make_unique<FastOS_File>(woFilename.c_str()); + ASSERT_TRUE(myFile->OpenWriteOnly()); + EXPECT_EQ(myFile->GetSize(), 0); + char dummyData[6] = "Dummy"; + ASSERT_TRUE(myFile->CheckedWrite(dummyData, 6)); + ASSERT_EQ(myFile->GetPosition(), 6); + ASSERT_TRUE(myFile->SetPosition(0)); + ASSERT_EQ(myFile->GetPosition(), 0); + EXPECT_LT(myFile->Read(dummyData, 6), 0); + EXPECT_TRUE(myFile->Close()); + EXPECT_TRUE(myFile->Delete()); +} + +TEST(FileTest, ReadWriteTest) { + Generated guard; + auto myFile = std::make_unique<FastOS_File>(rwFilename.c_str()); + ASSERT_FALSE(myFile->OpenExisting()); + ASSERT_TRUE(myFile->OpenReadWrite()); + ASSERT_EQ(myFile->GetSize(), 0); + char dummyData[6] = "Dummy"; + ASSERT_TRUE(myFile->CheckedWrite(dummyData, 6)); + ASSERT_EQ(myFile->GetPosition(), 6); + ASSERT_TRUE(myFile->SetPosition(0)); + ASSERT_EQ(myFile->GetPosition(), 0); + char dummyData2[7]; + ASSERT_EQ(myFile->Read(dummyData2, 6), 6); + EXPECT_EQ(memcmp(dummyData, dummyData2, 6), 0); + ASSERT_TRUE(myFile->SetPosition(1)); + EXPECT_EQ(myFile->Read(dummyData2, 7), 5); + EXPECT_EQ(myFile->GetPosition(), 6); + EXPECT_EQ(myFile->Read(dummyData2, 6), 0); + EXPECT_EQ(myFile->GetPosition(), 6); + EXPECT_TRUE(myFile->Close()); + EXPECT_TRUE(myFile->Delete()); +} + +TEST(FileTest, ScanDirectoryTest) { + auto scanDir = std::make_unique<FastOS_DirectoryScan>("."); + while (scanDir->ReadNext()) { + const char *name = scanDir->GetName(); + bool isDirectory = scanDir->IsDirectory(); + bool isRegular = scanDir->IsRegular(); + fprintf(stderr, "%-30s %s\n", name, isDirectory ? "DIR" : (isRegular ? "FILE" : "UNKN")); + } +} + +TEST(FileTest, ReadBufTest) { + FastOS_File file(roFilename.c_str()); + char buffer[20]; + ASSERT_TRUE(file.OpenReadOnly()); + EXPECT_EQ(file.GetPosition(), 0); + EXPECT_EQ(file.Read(buffer, 4), 4); + buffer[4] = '\0'; + EXPECT_EQ(file.GetPosition(), 4); + EXPECT_EQ(strcmp(buffer, "This"), 0); + file.ReadBuf(buffer, 6, 8); + buffer[6] = '\0'; + EXPECT_EQ(file.GetPosition(), 4); + EXPECT_EQ(strcmp(buffer, "a test"), 0); +} + +TEST(FileTest, DiskFreeSpaceTest) { + EXPECT_NE(FastOS_File::GetFreeDiskSpace(roFilename.c_str()), int64_t(-1)); + EXPECT_NE(FastOS_File::GetFreeDiskSpace("."), int64_t(-1)); +} + +TEST(FileTest, MaxLengthTest) { + int maxval = FastOS_File::GetMaximumFilenameLength("."); + EXPECT_GT(maxval, 5); + EXPECT_LT(maxval, (512*1024)); + maxval = FastOS_File::GetMaximumPathLength("."); + EXPECT_GT(maxval, 5); + EXPECT_LT(maxval, (512*1024)); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/fastos/src/tests/hello.txt b/vespalib/src/tests/fastos/hello.txt index 62a405393a6..62a405393a6 100644 --- a/fastos/src/tests/hello.txt +++ b/vespalib/src/tests/fastos/hello.txt diff --git a/vespalib/src/tests/net/tls/capabilities/capabilities_test.cpp b/vespalib/src/tests/net/tls/capabilities/capabilities_test.cpp index 0bf04289a65..5fdddf086ba 100644 --- a/vespalib/src/tests/net/tls/capabilities/capabilities_test.cpp +++ b/vespalib/src/tests/net/tls/capabilities/capabilities_test.cpp @@ -84,6 +84,7 @@ TEST("All known capabilities can be looked up by name, and resolve back to same check_capability_mapping("vespa.content.metrics_api", Capability::content_metrics_api()); check_capability_mapping("vespa.content.proton_admin_api", Capability::content_proton_admin_api()); check_capability_mapping("vespa.content.search_api", Capability::content_search_api()); + check_capability_mapping("vespa.content.state_api", Capability::content_state_api()); check_capability_mapping("vespa.content.status_pages", Capability::content_status_pages()); check_capability_mapping("vespa.content.storage_api", Capability::content_storage_api()); check_capability_mapping("vespa.logserver.api", Capability::logserver_api()); @@ -109,6 +110,7 @@ TEST("CapabilitySet instances can be stringified") { "vespa.container.state_api, " "vespa.content.document_api, " "vespa.content.metrics_api, " + "vespa.content.state_api, " "vespa.content.status_pages, " "vespa.content.storage_api, " "vespa.logserver.api, " @@ -124,7 +126,7 @@ TEST("All known capability sets can be looked up by name") { check_capability_set_mapping("vespa.telemetry", CapabilitySet::telemetry()); check_capability_set_mapping("vespa.cluster_controller_node", CapabilitySet::cluster_controller_node()); check_capability_set_mapping("vespa.logserver_node", CapabilitySet::logserver_node()); - check_capability_set_mapping("vespa.config_server", CapabilitySet::config_server()); + check_capability_set_mapping("vespa.config_server_node", CapabilitySet::config_server_node()); } TEST("Unknown capability set name returns nullopt") { @@ -135,7 +137,7 @@ TEST("Resolving a capability set adds all its underlying capabilities") { CapabilitySet caps; EXPECT_TRUE(caps.resolve_and_add("vespa.content_node")); // Slightly suboptimal; this test will fail if the default set of capabilities for vespa.content_node changes. - EXPECT_EQUAL(caps.count(), 14u); + EXPECT_EQUAL(caps.count(), 15u); EXPECT_FALSE(caps.empty()); EXPECT_TRUE(caps.contains(Capability::content_storage_api())); EXPECT_TRUE(caps.contains(Capability::content_document_api())); @@ -147,6 +149,7 @@ TEST("Resolving a capability set adds all its underlying capabilities") { EXPECT_TRUE(caps.contains(Capability::configproxy_config_api())); EXPECT_TRUE(caps.contains(Capability::configproxy_filedistribution_api())); // vespa.content_node -> shared node caps -> vespa.telemetry + EXPECT_TRUE(caps.contains(Capability::content_state_api())); EXPECT_TRUE(caps.contains(Capability::content_status_pages())); EXPECT_TRUE(caps.contains(Capability::content_metrics_api())); EXPECT_TRUE(caps.contains(Capability::container_state_api())); diff --git a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp index 0178443643e..068345b7254 100644 --- a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp +++ b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp @@ -735,6 +735,28 @@ TEST_F("Authz policy-derived peer capabilities are propagated to CryptoCodec", C Capability::content_status_pages()})); } +TEST_F("Handshake is allowed if at least one policy matches, even if resulting capability set is empty", CertFixture) { + auto server_ck = f.create_ca_issued_peer_cert({}, {{"DNS:hello.world.example.com"}}); + auto authorized = authorized_peers({policy_with({required_san_dns("stale.memes.example.com")}, + CapabilitySet::make_empty()), + policy_with({required_san_dns("fresh.memes.example.com")}, + CapabilitySet::make_with_all_capabilities())}); + f.reset_server_with_cert_opts(server_ck, std::move(authorized)); + auto client_ck = f.create_ca_issued_peer_cert({}, {{"DNS:stale.memes.example.com"}}); + f.reset_client_with_cert_opts(client_ck, AuthorizedPeers::allow_all_authenticated()); + + ASSERT_TRUE(f.handshake()); + + // Note: "inversion" of client <-> server is because the capabilities are that of the _peer_. + auto client_caps = f.server->granted_capabilities(); + auto server_caps = f.client->granted_capabilities(); + // Server (from client's PoV) implicitly has all capabilities since client doesn't specify any policies + EXPECT_EQUAL(server_caps, CapabilitySet::make_with_all_capabilities()); + // Client (from server's PoV) only has capabilities for the rule matching its DNS SAN entry. + // In this case, it is the empty set. + EXPECT_EQUAL(client_caps, CapabilitySet::make_empty()); +} + void reset_peers_with_server_authz_mode(CertFixture& f, AuthorizationMode authz_mode) { auto ck = f.create_ca_issued_peer_cert({"hello.world.example.com"}, {}); diff --git a/vespalib/src/tests/ref_counted/CMakeLists.txt b/vespalib/src/tests/ref_counted/CMakeLists.txt new file mode 100644 index 00000000000..74258dc5e67 --- /dev/null +++ b/vespalib/src/tests/ref_counted/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_ref_counted_test_app TEST + SOURCES + ref_counted_test.cpp + DEPENDS + vespalib + GTest::GTest +) +vespa_add_test(NAME vespalib_ref_counted_test_app COMMAND vespalib_ref_counted_test_app) diff --git a/vespalib/src/tests/ref_counted/ref_counted_test.cpp b/vespalib/src/tests/ref_counted/ref_counted_test.cpp new file mode 100644 index 00000000000..2a5321b7b27 --- /dev/null +++ b/vespalib/src/tests/ref_counted/ref_counted_test.cpp @@ -0,0 +1,257 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/util/ref_counted.h> +#include <vespa/vespalib/util/gate.h> +#include <vespa/vespalib/util/thread.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib; + +struct Base : enable_ref_counted { + static std::atomic<int> ctor_cnt; + static std::atomic<int> dtor_cnt; + int val; + Base(int val_in) : val(val_in) { + ctor_cnt.fetch_add(1, std::memory_order_relaxed); + } + ~Base() { + dtor_cnt.fetch_add(1, std::memory_order_relaxed); + } +}; +std::atomic<int> Base::ctor_cnt = 0; +std::atomic<int> Base::dtor_cnt = 0; + +struct Leaf : Base { + static std::atomic<int> ctor_cnt; + static std::atomic<int> dtor_cnt; + Leaf(int val_in) : Base(val_in) { + ctor_cnt.fetch_add(1, std::memory_order_relaxed); + } + ~Leaf() { + dtor_cnt.fetch_add(1, std::memory_order_relaxed); + } +}; +std::atomic<int> Leaf::ctor_cnt = 0; +std::atomic<int> Leaf::dtor_cnt = 0; + +void copy_assign_ref_counted_leaf_real(ref_counted<Leaf>& lhs, const ref_counted<Leaf> &rhs) +{ + lhs = rhs; +} + +void (*copy_assign_ref_counted_leaf)(ref_counted<Leaf>& lhs, const ref_counted<Leaf> &rhs) = copy_assign_ref_counted_leaf_real; + +void move_assign_ref_counted_leaf_real(ref_counted<Leaf>& lhs, ref_counted<Leaf>&& rhs) +{ + lhs = std::move(rhs); +} + +void (*move_assign_ref_counted_leaf)(ref_counted<Leaf>& lhs, ref_counted<Leaf>&& rhs) = move_assign_ref_counted_leaf_real; + +// check that the expected number of objects have been created and +// destroyed while this object was in scope. +struct CheckObjects { + int expect_base; + int expect_leaf; + int old_base_ctor; + int old_base_dtor; + int old_leaf_ctor; + int old_leaf_dtor; + CheckObjects(int expect_base_in = 0, int expect_leaf_in = 0) + : expect_base(expect_base_in), + expect_leaf(expect_leaf_in) + { + old_base_ctor = Base::ctor_cnt.load(std::memory_order_relaxed); + old_base_dtor = Base::dtor_cnt.load(std::memory_order_relaxed); + old_leaf_ctor = Leaf::ctor_cnt.load(std::memory_order_relaxed); + old_leaf_dtor = Leaf::dtor_cnt.load(std::memory_order_relaxed); + } + ~CheckObjects() { + int base_ctor_diff = Base::ctor_cnt.load(std::memory_order_relaxed) - old_base_ctor; + int base_dtor_diff = Base::dtor_cnt.load(std::memory_order_relaxed) - old_base_dtor; + int leaf_ctor_diff = Leaf::ctor_cnt.load(std::memory_order_relaxed) - old_leaf_ctor; + int leaf_dtor_diff = Leaf::dtor_cnt.load(std::memory_order_relaxed) - old_leaf_dtor; + EXPECT_EQ(base_ctor_diff, expect_base); + EXPECT_EQ(base_dtor_diff, expect_base); + EXPECT_EQ(leaf_ctor_diff, expect_leaf); + EXPECT_EQ(leaf_dtor_diff, expect_leaf); + } +}; + +TEST(RefCountedTest, create_empty_ref_counted) { + CheckObjects check; + ref_counted<Base> empty; + EXPECT_FALSE(empty); +} + +TEST(RefCountedTest, make_ref_counted) { + CheckObjects check(2, 1); + auto ref1 = make_ref_counted<Base>(10); + static_assert(std::same_as<decltype(ref1),ref_counted<Base>>); + EXPECT_TRUE(ref1); + EXPECT_EQ((*ref1).val, 10); + EXPECT_EQ(ref1->val, 10); + auto ref2 = make_ref_counted<Leaf>(20); + static_assert(std::same_as<decltype(ref2),ref_counted<Leaf>>); + EXPECT_TRUE(ref2); + EXPECT_EQ((*ref2).val, 20); + EXPECT_EQ(ref2->val, 20); +} + +TEST(RefCountedTest, ref_counted_from) { + CheckObjects check(1, 1); + auto ref = make_ref_counted<Leaf>(10); + Leaf &leaf = *ref; + Base &base = leaf; + EXPECT_EQ(ref->count_refs(), 1); + auto from_leaf = ref_counted_from(leaf); + static_assert(std::same_as<decltype(from_leaf),ref_counted<Leaf>>); + auto from_base = ref_counted_from(base); + static_assert(std::same_as<decltype(from_base),ref_counted<Base>>); + EXPECT_EQ(ref->count_refs(), 3); + EXPECT_EQ(from_base->val, 10); +} + +TEST(RefCountedTest, use_internal_api) { + CheckObjects check(1); + Base *raw = new Base(20); + EXPECT_EQ(raw->count_refs(), 1); + ref_counted<Base> ref = ref_counted<Base>::internal_attach(raw); + EXPECT_EQ(ref->count_refs(), 1); + EXPECT_EQ(ref->val, 20); + EXPECT_EQ(ref.internal_detach(), raw); + EXPECT_EQ(raw->count_refs(), 1); + raw->internal_addref(); + EXPECT_EQ(raw->count_refs(), 2); + raw->internal_subref(); + EXPECT_EQ(raw->count_refs(), 1); + raw->internal_subref(); +} + +TEST(RefCountedTest, use_multi_ref_internal_api) { + CheckObjects check(1); + Base *raw = new Base(20); + EXPECT_EQ(raw->count_refs(), 1); + raw->internal_addref(9); + EXPECT_EQ(raw->count_refs(), 10); + EXPECT_EQ(raw->val, 20); + raw->internal_subref(6, 4); + EXPECT_EQ(raw->count_refs(), 4); + raw->internal_subref(4, 0); +} + +TEST(RefCountedTest, move_ref_counted) { + for (bool has_src: {true, false}) { + for (bool has_dst: {true, false}) { + for (bool same: {true, false}) { + if (same) { + CheckObjects check(has_src + has_dst); + ref_counted<Base> src = has_src ? make_ref_counted<Base>(10) : ref_counted<Base>(); + ref_counted<Base> dst = has_dst ? make_ref_counted<Base>(20) : ref_counted<Base>(); + dst = std::move(src); + EXPECT_EQ(dst, has_src); + EXPECT_FALSE(src); + if (has_src) { + EXPECT_EQ(dst->val, 10); + EXPECT_EQ(dst->count_refs(), 1); + } + } else { + CheckObjects check(has_src + has_dst, has_src + has_dst); + ref_counted<Leaf> src = has_src ? make_ref_counted<Leaf>(10) : ref_counted<Leaf>(); + ref_counted<Base> dst = has_dst ? make_ref_counted<Leaf>(20) : ref_counted<Leaf>(); + dst = std::move(src); + EXPECT_EQ(dst, has_src); + EXPECT_FALSE(src); + if (has_src) { + EXPECT_EQ(dst->val, 10); + EXPECT_EQ(dst->count_refs(), 1); + } + } + } + } + } +} + +TEST(RefCountedTest, copy_ref_counted) { + for (bool has_src: {true, false}) { + for (bool has_dst: {true, false}) { + for (bool same: {true, false}) { + if (same) { + CheckObjects check(2); + ref_counted<Base> empty; + ref_counted<Base> obj1 = make_ref_counted<Base>(10); + ref_counted<Base> obj2 = make_ref_counted<Base>(20); + ref_counted<Base> src = has_src ? obj1 : empty; + ref_counted<Base> dst = has_dst ? obj2 : empty; + dst = src; + EXPECT_EQ(dst, has_src); + EXPECT_EQ(src, has_src); + if (has_src) { + EXPECT_EQ(dst->val, 10); + EXPECT_EQ(dst->count_refs(), 3); + } + } else { + CheckObjects check(2, 2); + ref_counted<Leaf> empty; + ref_counted<Leaf> obj1 = make_ref_counted<Leaf>(10); + ref_counted<Leaf> obj2 = make_ref_counted<Leaf>(20); + ref_counted<Leaf> src = has_src ? obj1 : empty; + ref_counted<Base> dst = has_dst ? obj2 : empty; + dst = src; + EXPECT_EQ(dst, has_src); + EXPECT_EQ(src, has_src); + if (has_src) { + EXPECT_EQ(dst->val, 10); + EXPECT_EQ(dst->count_refs(), 3); + } + } + } + } + } +} + +struct Other : enable_ref_counted {}; + +TEST(RefCountedTest, compile_errors_when_uncommented) { + struct Foo {}; + [[maybe_unused]] Foo foo; + [[maybe_unused]] ref_counted<Other> other = make_ref_counted<Other>(); + // ref_counted<Foo> empty; + // auto ref1 = make_ref_counted<Foo>(); + // auto ref2 = ref_counted_from(foo); + // ref_counted<Base> base = other; +} + +TEST(RefCountedTest, self_assign) { + ref_counted<Leaf> ref = make_ref_counted<Leaf>(10); + copy_assign_ref_counted_leaf(ref, ref); + move_assign_ref_counted_leaf(ref, std::move(ref)); + EXPECT_EQ(ref->count_refs(), 1); + EXPECT_EQ(ref->val, 10); +} + +TEST(RefCountedTest, with_threads) { + CheckObjects check(2,1); + ThreadPool pool; + Gate gate; + { + auto a = make_ref_counted<Base>(10); + auto b = make_ref_counted<Leaf>(20); + for (int i = 0; i < 8; ++i) { + pool.start([&gate,a,b]() + { + gate.await(); + for (int j = 0; j < 100000; ++j) { + auto c = a; + auto d = b; + EXPECT_EQ(c->val, 10); + EXPECT_EQ(d->val, 20); + } + }); + } + } + gate.countDown(); + pool.join(); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/referencecounter/.gitignore b/vespalib/src/tests/referencecounter/.gitignore deleted file mode 100644 index 7c753a29fab..00000000000 --- a/vespalib/src/tests/referencecounter/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -referencecounter_test -vespalib_referencecounter_test_app diff --git a/vespalib/src/tests/referencecounter/CMakeLists.txt b/vespalib/src/tests/referencecounter/CMakeLists.txt deleted file mode 100644 index ee909b48281..00000000000 --- a/vespalib/src/tests/referencecounter/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vespalib_referencecounter_test_app TEST - SOURCES - referencecounter_test.cpp - DEPENDS - vespalib -) -vespa_add_test(NAME vespalib_referencecounter_test_app COMMAND vespalib_referencecounter_test_app) diff --git a/vespalib/src/tests/referencecounter/referencecounter_test.cpp b/vespalib/src/tests/referencecounter/referencecounter_test.cpp deleted file mode 100644 index 99dd455bcc8..00000000000 --- a/vespalib/src/tests/referencecounter/referencecounter_test.cpp +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("referencecounter_test"); -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/referencecounter.h> - -struct Data -{ - int ctorCnt; - int dtorCnt; - Data() : ctorCnt(0), dtorCnt(0) {} -}; - -class DataRef : public vespalib::ReferenceCounter -{ -private: - Data &_d; - DataRef(const DataRef &); - DataRef &operator=(const DataRef &); -public: - DataRef(Data &d) : _d(d) { ++d.ctorCnt; } - ~DataRef() { ++_d.dtorCnt; } - int getCtorCnt() const { return _d.ctorCnt; } - int getDtorCnt() const { return _d.dtorCnt; } -}; - -using namespace vespalib; - -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("referencecounter_test"); - - Data data; - { - DataRef *pt1 = new DataRef(data); - - EXPECT_TRUE(pt1->refCount() == 1); - - DataRef *pt2 = pt1; - pt2->addRef(); - - EXPECT_TRUE(pt1->refCount() == 2); - - EXPECT_TRUE(data.ctorCnt == 1); - EXPECT_TRUE(data.dtorCnt == 0); - pt1->subRef(); - EXPECT_TRUE(pt1->refCount() == 1); - pt2->subRef(); - } - EXPECT_TRUE(data.ctorCnt == 1); - EXPECT_TRUE(data.dtorCnt == 1); - TEST_DONE(); -} diff --git a/vespalib/src/tests/thread/thread_test.cpp b/vespalib/src/tests/thread/thread_test.cpp index 9533fe1c190..077b85ea1ac 100644 --- a/vespalib/src/tests/thread/thread_test.cpp +++ b/vespalib/src/tests/thread/thread_test.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/thread.h> +#include <iostream> using namespace vespalib; @@ -11,6 +12,7 @@ struct Agent : public Runnable { bool was_run; Agent() : was_run(false) {} void run() override { + fprintf(stderr, "agent run in thread %zu\n", thread::as_zu(std::this_thread::get_id())); was_run = true; } }; @@ -22,11 +24,18 @@ void my_fun(bool *was_run) { Runnable::init_fun_t wrap(Runnable::init_fun_t init, bool *init_called) { return [=](Runnable &target) { + fprintf(stderr, "lambda run in thread %zu\n", thread::as_zu(std::this_thread::get_id())); *init_called = true; return init(target); }; } +TEST("main thread") { + auto my_id = std::this_thread::get_id(); + std::cerr << "main thread(with <<): " << my_id << "\n"; + fprintf(stderr, "main thread(with printf): %zu\n", thread::as_zu(my_id)); +} + TEST("run vespalib::Runnable with init function") { Agent agent; bool init_called = false; @@ -41,9 +50,17 @@ TEST("use thread pool to run multiple things") { bool init_called = false; bool was_run = false; ThreadPool pool; + EXPECT_TRUE(pool.empty()); + EXPECT_EQUAL(pool.size(), 0u); pool.start(my_fun, &was_run); + EXPECT_TRUE(!pool.empty()); + EXPECT_EQUAL(pool.size(), 1u); pool.start(agent, wrap(test_agent_thread, &init_called)); + EXPECT_TRUE(!pool.empty()); + EXPECT_EQUAL(pool.size(), 2u); pool.join(); + EXPECT_TRUE(pool.empty()); + EXPECT_EQUAL(pool.size(), 0u); EXPECT_TRUE(init_called); EXPECT_TRUE(agent.was_run); EXPECT_TRUE(was_run); diff --git a/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp b/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp index 1cc54da7f2e..4abae54e06f 100644 --- a/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp +++ b/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp @@ -1,12 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("generation_handler_stress_test"); + #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/generationhandler.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/size_literals.h> #include <thread> +#include <cinttypes> + +#include <vespa/log/log.h> +LOG_SETUP("generation_handler_stress_test"); using vespalib::Executor; using vespalib::GenerationHandler; diff --git a/vespalib/src/vespa/fastlib/io/CMakeLists.txt b/vespalib/src/vespa/fastlib/io/CMakeLists.txt index f21cf27b21e..9f6211f3e19 100644 --- a/vespalib/src/vespa/fastlib/io/CMakeLists.txt +++ b/vespalib/src/vespa/fastlib/io/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(fastlib_io OBJECT +vespa_add_library(vespalib_fastlib_io OBJECT SOURCES bufferedfile.cpp DEPENDS diff --git a/vespalib/src/vespa/fastlib/io/bufferedfile.cpp b/vespalib/src/vespa/fastlib/io/bufferedfile.cpp index 31ca735a0bb..aecf08edf6b 100644 --- a/vespalib/src/vespa/fastlib/io/bufferedfile.cpp +++ b/vespalib/src/vespa/fastlib/io/bufferedfile.cpp @@ -107,13 +107,6 @@ Fast_BufferedFile::Sync() return _file->Sync(); } -time_t -Fast_BufferedFile::GetModificationTime() -{ - time_t retval = _file->GetModificationTime(); - return retval; -} - void Fast_BufferedFile::EnableDirectIO() { diff --git a/vespalib/src/vespa/fastlib/io/bufferedfile.h b/vespalib/src/vespa/fastlib/io/bufferedfile.h index 48f90262ad9..2a5e0ec7535 100644 --- a/vespalib/src/vespa/fastlib/io/bufferedfile.h +++ b/vespalib/src/vespa/fastlib/io/bufferedfile.h @@ -172,12 +172,7 @@ public: * Force completion of pending disk writes (flush cache). */ [[nodiscard]] bool Sync() override; - /** - * Get the time the file was last modified. - * - * @return time_t The last modification time. - */ - time_t GetModificationTime() override; + /** * Turn on direct IO. */ diff --git a/vespalib/src/vespa/fastlib/text/CMakeLists.txt b/vespalib/src/vespa/fastlib/text/CMakeLists.txt index d6cb8c29305..06573ee814d 100644 --- a/vespalib/src/vespa/fastlib/text/CMakeLists.txt +++ b/vespalib/src/vespa/fastlib/text/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(fastlib_text OBJECT +vespa_add_library(vespalib_fastlib_text OBJECT SOURCES unicodeutil.cpp normwordfolder.cpp diff --git a/vespalib/src/vespa/fastos/CMakeLists.txt b/vespalib/src/vespa/fastos/CMakeLists.txt new file mode 100644 index 00000000000..2b7a2c3b905 --- /dev/null +++ b/vespalib/src/vespa/fastos/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(vespalib_fastos OBJECT + SOURCES + file.cpp + file_rw_ops.cpp + linux_file.cpp + unix_file.cpp + DEPENDS +) diff --git a/fastos/src/vespa/fastos/file.cpp b/vespalib/src/vespa/fastos/file.cpp index fdbacb570b4..fdbacb570b4 100644 --- a/fastos/src/vespa/fastos/file.cpp +++ b/vespalib/src/vespa/fastos/file.cpp diff --git a/fastos/src/vespa/fastos/file.h b/vespalib/src/vespa/fastos/file.h index 1cf6fee71dd..1a637726e45 100644 --- a/fastos/src/vespa/fastos/file.h +++ b/vespalib/src/vespa/fastos/file.h @@ -10,10 +10,11 @@ #pragma once -#include "types.h" -#include <cstdint> +#include <vespa/vespalib/util/time.h> #include <string> +#define FASTOS_PREFIX(a) FastOS_##a + constexpr int FASTOS_FILE_OPEN_READ = (1<<0); constexpr int FASTOS_FILE_OPEN_WRITE = (1<<1); constexpr int FASTOS_FILE_OPEN_EXISTING = (1<<2); @@ -100,12 +101,6 @@ public: void setFAdviseOptions(int options) { _fAdviseOptions = options; } /** - * Return path separator string. This will yield "/" on UNIX systems. - * @return pointer to path separator character string - */ - static const char *GetPathSeparator() { return "/"; } - - /** * Constructor. A filename could be supplied at this point, or specified * later using @ref SetFileName() or @ref Open(). * @param filename a filename (optional) @@ -343,12 +338,6 @@ public: int64_t getSize() const { return const_cast<FastOS_FileInterface *>(this)->GetSize(); } /** - * Return the time when file was last modified. - * @return time of last modification - */ - virtual time_t GetModificationTime() = 0; - - /** * Delete the file. This method requires that the file is * currently not opened. * @return Boolean success/failure @@ -412,10 +401,6 @@ public: bool useSyncWrites() const { return _syncWritesEnabled; } - /** - * Set the write chunk size used in WriteBuf. - */ - void setChunkSize(size_t chunkSize) { _chunkSize = chunkSize; } size_t getChunkSize() const { return _chunkSize; } /** @@ -612,12 +597,7 @@ public: /** * Time of last modification in seconds. */ - time_t _modifiedTime; - - /** - * Time of last modification in seconds. - */ - uint64_t _modifiedTimeNS; + vespalib::system_time _modifiedTime; }; @@ -651,14 +631,12 @@ public: */ class FastOS_DirectoryScanInterface { -private: - FastOS_DirectoryScanInterface(const FastOS_DirectoryScanInterface&); - FastOS_DirectoryScanInterface& operator= (const FastOS_DirectoryScanInterface&); - protected: std::string _searchPath; public: + FastOS_DirectoryScanInterface(const FastOS_DirectoryScanInterface&) = delete; + FastOS_DirectoryScanInterface& operator= (const FastOS_DirectoryScanInterface&) = delete; /** * Constructor. @@ -676,13 +654,6 @@ public: virtual ~FastOS_DirectoryScanInterface(); /** - * Get search path. - * This is an internal copy of the path specified in the constructor. - * @return Search path string. - */ - const char *GetSearchPath () { return _searchPath.c_str(); } - - /** * Read the next entry in the directory scan. Failure indicates * that there are no more entries. If the call is successful, * attributes for the entry can be read with @ref IsDirectory(), @@ -725,14 +696,6 @@ public: * @return A pointer to the recently read directory entry. */ virtual const char *GetName() = 0; - - /** - * Check whether the creation of a directory scan succeeded or - * failed (e.g. due to resource constraints). - * - * return True if the directory scan is valid. - */ - virtual bool IsValidScan() const = 0; }; #ifdef __linux__ diff --git a/fastos/src/vespa/fastos/file_rw_ops.cpp b/vespalib/src/vespa/fastos/file_rw_ops.cpp index 79fe95b21f2..79fe95b21f2 100644 --- a/fastos/src/vespa/fastos/file_rw_ops.cpp +++ b/vespalib/src/vespa/fastos/file_rw_ops.cpp diff --git a/fastos/src/vespa/fastos/file_rw_ops.h b/vespalib/src/vespa/fastos/file_rw_ops.h index 4f7aa6f082f..4f7aa6f082f 100644 --- a/fastos/src/vespa/fastos/file_rw_ops.h +++ b/vespalib/src/vespa/fastos/file_rw_ops.h diff --git a/fastos/src/vespa/fastos/linux_file.cpp b/vespalib/src/vespa/fastos/linux_file.cpp index 6fb29782957..6fb29782957 100644 --- a/fastos/src/vespa/fastos/linux_file.cpp +++ b/vespalib/src/vespa/fastos/linux_file.cpp diff --git a/fastos/src/vespa/fastos/linux_file.h b/vespalib/src/vespa/fastos/linux_file.h index 2481b163210..2481b163210 100644 --- a/fastos/src/vespa/fastos/linux_file.h +++ b/vespalib/src/vespa/fastos/linux_file.h diff --git a/fastos/src/vespa/fastos/unix_file.cpp b/vespalib/src/vespa/fastos/unix_file.cpp index 7c4cde19125..802e85d7609 100644 --- a/fastos/src/vespa/fastos/unix_file.cpp +++ b/vespalib/src/vespa/fastos/unix_file.cpp @@ -29,6 +29,10 @@ using fastos::File_RW_Ops; +namespace { + constexpr uint64_t ONE_G = 1000 * 1000 * 1000; +} + int FastOS_UNIX_File::GetLastOSError() { return errno; } @@ -61,8 +65,7 @@ FastOS_UNIX_File::GetPosition() return lseek(_filedes, 0, SEEK_CUR); } -void FastOS_UNIX_File::ReadBuf(void *buffer, size_t length, - int64_t readOffset) +void FastOS_UNIX_File::ReadBuf(void *buffer, size_t length, int64_t readOffset) { ssize_t readResult; @@ -94,19 +97,13 @@ FastOS_UNIX_File::Stat(const char *filename, FastOS_StatInfo *statInfo) statInfo->_isRegular = S_ISREG(stbuf.st_mode); statInfo->_isDirectory = S_ISDIR(stbuf.st_mode); statInfo->_size = static_cast<int64_t>(stbuf.st_size); - statInfo->_modifiedTime = stbuf.st_mtime; + uint64_t modTimeNS = stbuf.st_mtime * ONE_G; #ifdef __linux__ - statInfo->_modifiedTimeNS = stbuf.st_mtim.tv_sec; - statInfo->_modifiedTimeNS *= 1000000000; - statInfo->_modifiedTimeNS += stbuf.st_mtim.tv_nsec; + modTimeNS += stbuf.st_mtim.tv_nsec; #elif defined(__APPLE__) - statInfo->_modifiedTimeNS = stbuf.st_mtimespec.tv_sec; - statInfo->_modifiedTimeNS *= 1000000000; - statInfo->_modifiedTimeNS += stbuf.st_mtimespec.tv_nsec; -#else - statInfo->_modifiedTimeNS = stbuf.st_mtime; - statInfo->_modifiedTimeNS *= 1000000000; + modTimeNS += stbuf.st_mtimespec.tv_nsec; #endif + statInfo->_modifiedTime = vespalib::system_time(std::chrono::duration_cast<vespalib::system_time::duration>(std::chrono::nanoseconds(modTimeNS))); rc = true; } else { if (errno == ENOENT) { @@ -344,21 +341,6 @@ FastOS_UNIX_File::GetSize() return fileSize; } - -time_t -FastOS_UNIX_File::GetModificationTime() -{ - struct stat stbuf{}; - assert(IsOpened()); - - int res = fstat(_filedes, &stbuf); - assert(res == 0); - (void) res; - - return stbuf.st_mtime; -} - - bool FastOS_UNIX_File::Delete(const char *name) { @@ -581,10 +563,3 @@ FastOS_UNIX_DirectoryScan::GetName() return static_cast<const char *>(_dp->d_name); } - - -bool -FastOS_UNIX_DirectoryScan::IsValidScan() const -{ - return _dir != nullptr; -} diff --git a/fastos/src/vespa/fastos/unix_file.h b/vespalib/src/vespa/fastos/unix_file.h index 31e45f8d2fa..3d1f6b9db3f 100644 --- a/fastos/src/vespa/fastos/unix_file.h +++ b/vespalib/src/vespa/fastos/unix_file.h @@ -82,7 +82,6 @@ public: bool SetPosition(int64_t desiredPosition) override; int64_t GetPosition() override; int64_t GetSize() override; - time_t GetModificationTime() override; bool Delete() override; [[nodiscard]] bool Sync() override; bool SetSize(int64_t newSize) override; @@ -103,9 +102,6 @@ public: class FastOS_UNIX_DirectoryScan : public FastOS_DirectoryScanInterface { private: - FastOS_UNIX_DirectoryScan(const FastOS_UNIX_DirectoryScan&); - FastOS_UNIX_DirectoryScan& operator=(const FastOS_UNIX_DirectoryScan&); - bool _statRun; bool _isDirectory; bool _isRegular; @@ -126,5 +122,4 @@ public: bool IsDirectory() override; bool IsRegular() override; const char *GetName() override; - bool IsValidScan() const override; }; diff --git a/vespalib/src/vespa/vespalib/CMakeLists.txt b/vespalib/src/vespa/vespalib/CMakeLists.txt index 9e31084c067..63876d442ef 100644 --- a/vespalib/src/vespa/vespalib/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/CMakeLists.txt @@ -30,8 +30,9 @@ vespa_add_library(vespalib $<TARGET_OBJECTS:vespalib_vespalib_time> $<TARGET_OBJECTS:vespalib_vespalib_trace> $<TARGET_OBJECTS:vespalib_vespalib_util> - $<TARGET_OBJECTS:fastlib_io> - $<TARGET_OBJECTS:fastlib_text> + $<TARGET_OBJECTS:vespalib_fastlib_io> + $<TARGET_OBJECTS:vespalib_fastlib_text> + $<TARGET_OBJECTS:vespalib_fastos> INSTALL lib64 DEPENDS ${VESPA_GCC_LIB} diff --git a/vespalib/src/vespa/vespalib/data/slime/json_format.cpp b/vespalib/src/vespa/vespalib/data/slime/json_format.cpp index 2681ba6f52b..9cda8a7fae1 100644 --- a/vespalib/src/vespa/vespalib/data/slime/json_format.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/json_format.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/data/memory_input.h> #include <vespa/vespalib/locale/c.h> #include <cmath> +#include <cinttypes> #include <sstream> #include <cassert> diff --git a/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt b/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt index f11004363f8..1c3b9112dda 100644 --- a/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(vespalib_vespalib_datastore OBJECT SOURCES array_store.cpp array_store_config.cpp + array_store_type_mapper.cpp atomic_entry_ref.cpp buffer_free_list.cpp buffer_stats.cpp diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.h b/vespalib/src/vespa/vespalib/datastore/array_store.h index 9e403a52b44..2a193172959 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store.h @@ -2,7 +2,7 @@ #pragma once -#include "array_store_type_mapper.h" +#include "array_store_simple_type_mapper.h" #include "array_store_config.h" #include "buffer_type.h" #include "bufferstate.h" @@ -29,7 +29,7 @@ namespace vespalib::datastore { * * The max value of maxSmallArrayTypeId is (2^bufferBits - 1). */ -template <typename EntryT, typename RefT = EntryRefT<19>, typename TypeMapperT = ArrayStoreTypeMapper<EntryT> > +template <typename EntryT, typename RefT = EntryRefT<19>, typename TypeMapperT = ArrayStoreSimpleTypeMapper<EntryT> > class ArrayStore : public ICompactable { public: diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_simple_type_mapper.h b/vespalib/src/vespa/vespalib/datastore/array_store_simple_type_mapper.h new file mode 100644 index 00000000000..a0cd7827b2d --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/array_store_simple_type_mapper.h @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "large_array_buffer_type.h" +#include "small_array_buffer_type.h" + +namespace vespalib::datastore { + +/** + * This class provides a 1-to-1 mapping between type ids and array sizes for small arrays in an array store. + * + * This means that buffers for type id 1 stores arrays of size 1, buffers for type id 2 stores arrays of size 2, and so on. + * Type id 0 is always reserved for large arrays allocated on the heap. + * + * A more complex mapping can be used by creating a custom mapper and BufferType implementations. + */ +template <typename EntryT> +class ArrayStoreSimpleTypeMapper { +public: + using SmallBufferType = SmallArrayBufferType<EntryT>; + using LargeBufferType = LargeArrayBufferType<EntryT>; + + uint32_t get_type_id(size_t array_size) const { return array_size; } + size_t get_array_size(uint32_t type_id) const { return type_id; } + static uint32_t get_max_small_array_type_id(uint32_t max_small_array_type_id) noexcept { return max_small_array_type_id; } +}; + +} diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_type_mapper.cpp b/vespalib/src/vespa/vespalib/datastore/array_store_type_mapper.cpp new file mode 100644 index 00000000000..ff514f5a00b --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/array_store_type_mapper.cpp @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "array_store_type_mapper.h" +#include <algorithm> +#include <cassert> + +namespace vespalib::datastore { + +ArrayStoreTypeMapper::ArrayStoreTypeMapper() + : _array_sizes() +{ +} + +ArrayStoreTypeMapper::~ArrayStoreTypeMapper() = default; + +uint32_t +ArrayStoreTypeMapper::get_type_id(size_t array_size) const +{ + assert(!_array_sizes.empty()); + auto result = std::lower_bound(_array_sizes.begin() + 1, _array_sizes.end(), array_size); + if (result == _array_sizes.end()) { + return 0; // type id 0 uses buffer type for large arrays + } + return result - _array_sizes.begin(); +} + +size_t +ArrayStoreTypeMapper::get_array_size(uint32_t type_id) const +{ + assert(type_id > 0 && type_id < _array_sizes.size()); + return _array_sizes[type_id]; +} + +uint32_t +ArrayStoreTypeMapper::get_max_small_array_type_id(uint32_t max_small_array_type_id) const noexcept +{ + auto clamp_type_id = _array_sizes.size() - 1; + return (clamp_type_id < max_small_array_type_id) ? clamp_type_id : max_small_array_type_id; +} + +} diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_type_mapper.h b/vespalib/src/vespa/vespalib/datastore/array_store_type_mapper.h index 1b1e085ed16..e707627de19 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store_type_mapper.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store_type_mapper.h @@ -2,28 +2,30 @@ #pragma once -#include "large_array_buffer_type.h" -#include "small_array_buffer_type.h" +#include <cstddef> +#include <cstdint> +#include <vector> namespace vespalib::datastore { -/** - * This class provides a 1-to-1 mapping between type ids and array sizes for small arrays in an array store. +/* + * This class provides mapping between type ids and array sizes needed for + * storing a value with size smaller than or equal to the array size. * - * This means that buffers for type id 1 stores arrays of size 1, buffers for type id 2 stores arrays of size 2, and so on. - * Type id 0 is always reserved for large arrays allocated on the heap. - * - * A more complex mapping can be used by creating a custom mapper and BufferType implementations. + * The array sizes vector is a monotic increasing sequence that might end + * with exponential growth. */ -template <typename EntryT> -class ArrayStoreTypeMapper { +class ArrayStoreTypeMapper +{ +protected: + std::vector<size_t> _array_sizes; public: - using SmallBufferType = SmallArrayBufferType<EntryT>; - using LargeBufferType = LargeArrayBufferType<EntryT>; + ArrayStoreTypeMapper(); + ~ArrayStoreTypeMapper(); - uint32_t get_type_id(size_t array_size) const { return array_size; } - size_t get_array_size(uint32_t type_id) const { return type_id; } - static uint32_t get_max_small_array_type_id(uint32_t max_small_array_type_id) noexcept { return max_small_array_type_id; } + uint32_t get_type_id(size_t array_size) const; + size_t get_array_size(uint32_t type_id) const; + uint32_t get_max_small_array_type_id(uint32_t max_small_array_type_id) const noexcept; }; } diff --git a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h index 87083b2a878..600925969a3 100644 --- a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h +++ b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h @@ -11,8 +11,6 @@ namespace vespalib::alloc { class MemoryAllocator; } namespace vespalib::datastore { -template <typename EntryT> class ArrayStoreTypeMapper; - /* * Class representing buffer type for large arrays in ArrayStore */ @@ -26,7 +24,12 @@ class LargeArrayBufferType : public BufferType<Array<EntryT>> using CleanContext = typename ParentType::CleanContext; std::shared_ptr<alloc::MemoryAllocator> _memory_allocator; public: - LargeArrayBufferType(const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator, ArrayStoreTypeMapper<EntryT>& mapper) noexcept; + LargeArrayBufferType(const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) noexcept; + template <typename TypeMapper> + LargeArrayBufferType(const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator, TypeMapper&) noexcept + : LargeArrayBufferType(spec, std::move(memory_allocator)) + { + } ~LargeArrayBufferType() override; void cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; diff --git a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp index f4d1d1c90a1..3042bbff73f 100644 --- a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp +++ b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp @@ -8,7 +8,7 @@ namespace vespalib::datastore { template <typename EntryT> -LargeArrayBufferType<EntryT>::LargeArrayBufferType(const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator, ArrayStoreTypeMapper<EntryT>&) noexcept +LargeArrayBufferType<EntryT>::LargeArrayBufferType(const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) noexcept : BufferType<Array<EntryT>>(1u, spec.minArraysInBuffer, spec.maxArraysInBuffer, spec.numArraysForNewBuffer, spec.allocGrowFactor), _memory_allocator(std::move(memory_allocator)) { diff --git a/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h index f6cb860348d..676a9d3790f 100644 --- a/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h +++ b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h @@ -10,8 +10,6 @@ namespace vespalib::alloc { class MemoryAllocator; } namespace vespalib::datastore { -template <typename EntryT> class ArrayStoreTypeMapper; - /* * Class representing buffer type for small arrays in ArrayStore */ @@ -25,7 +23,12 @@ public: SmallArrayBufferType& operator=(const SmallArrayBufferType&) = delete; SmallArrayBufferType(SmallArrayBufferType&&) noexcept = default; SmallArrayBufferType& operator=(SmallArrayBufferType&&) noexcept = default; - SmallArrayBufferType(uint32_t array_size, const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator, ArrayStoreTypeMapper<EntryT>& mapper) noexcept; + SmallArrayBufferType(uint32_t array_size, const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) noexcept; + template <typename TypeMapper> + SmallArrayBufferType(uint32_t array_size, const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator, TypeMapper&) noexcept + : SmallArrayBufferType(array_size, spec, std::move(memory_allocator)) + { + } ~SmallArrayBufferType() override; const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; }; diff --git a/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp index 9068b8db7b1..414804417eb 100644 --- a/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp +++ b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp @@ -7,7 +7,7 @@ namespace vespalib::datastore { template <typename EntryT> -SmallArrayBufferType<EntryT>::SmallArrayBufferType(uint32_t array_size, const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator, ArrayStoreTypeMapper<EntryT>&) noexcept +SmallArrayBufferType<EntryT>::SmallArrayBufferType(uint32_t array_size, const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) noexcept : BufferType<EntryT>(array_size, spec.minArraysInBuffer, spec.maxArraysInBuffer, spec.numArraysForNewBuffer, spec.allocGrowFactor), _memory_allocator(std::move(memory_allocator)) { diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp b/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp index f5d87e14802..3bdbb7a81ff 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp @@ -3,7 +3,6 @@ #pragma once #include "private_helpers.hpp" -#include <vespa/fastos/types.h> namespace vespalib::hwaccelrated::avx { diff --git a/vespalib/src/vespa/vespalib/net/native_epoll.cpp b/vespalib/src/vespa/vespalib/net/native_epoll.cpp index 1e68826a145..8b73b092ab2 100644 --- a/vespalib/src/vespa/vespalib/net/native_epoll.cpp +++ b/vespalib/src/vespa/vespalib/net/native_epoll.cpp @@ -3,6 +3,7 @@ #include "native_epoll.h" #include <cassert> #include <cerrno> +#include <cstring> #include <unistd.h> #include <vespa/log/log.h> diff --git a/vespalib/src/vespa/vespalib/net/tls/capability.cpp b/vespalib/src/vespa/vespalib/net/tls/capability.cpp index cfc1cc7a7cc..49f8aa11bad 100644 --- a/vespalib/src/vespa/vespalib/net/tls/capability.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/capability.cpp @@ -35,6 +35,7 @@ constexpr std::array<std::string_view, Capability::max_value_count()> capability "vespa.content.metrics_api"sv, "vespa.content.proton_admin_api"sv, "vespa.content.search_api"sv, + "vespa.content.state_api"sv, "vespa.content.status_pages"sv, "vespa.content.storage_api"sv, "vespa.logserver.api"sv, @@ -83,6 +84,7 @@ std::optional<Capability> Capability::find_capability(const string& cap_name) no {"vespa.content.metrics_api", content_metrics_api()}, {"vespa.content.proton_admin_api", content_proton_admin_api()}, {"vespa.content.search_api", content_search_api()}, + {"vespa.content.state_api", content_state_api()}, {"vespa.content.status_pages", content_status_pages()}, {"vespa.content.storage_api", content_storage_api()}, {"vespa.logserver.api", logserver_api()}, diff --git a/vespalib/src/vespa/vespalib/net/tls/capability.h b/vespalib/src/vespa/vespalib/net/tls/capability.h index a7a1dcd15ac..396fad4cbcd 100644 --- a/vespalib/src/vespa/vespalib/net/tls/capability.h +++ b/vespalib/src/vespa/vespalib/net/tls/capability.h @@ -47,6 +47,7 @@ private: ContentMetricsApi, ContentProtonAdminApi, ContentSearchApi, + ContentStateApi, ContentStatusPages, ContentStorageApi, LogserverApi, @@ -176,6 +177,10 @@ public: return Capability(Id::ContentSearchApi); } + constexpr static Capability content_state_api() noexcept { + return Capability(Id::ContentStateApi); + } + constexpr static Capability content_proton_admin_api() noexcept { return Capability(Id::ContentProtonAdminApi); } diff --git a/vespalib/src/vespa/vespalib/net/tls/capability_set.cpp b/vespalib/src/vespa/vespalib/net/tls/capability_set.cpp index 3c457e96583..06231582461 100644 --- a/vespalib/src/vespa/vespalib/net/tls/capability_set.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/capability_set.cpp @@ -32,7 +32,7 @@ std::optional<CapabilitySet> CapabilitySet::find_capability_set(const string& ca {"vespa.telemetry", telemetry()}, {"vespa.cluster_controller_node", cluster_controller_node()}, {"vespa.logserver_node", logserver_node()}, - {"vespa.config_server", config_server()} + {"vespa.config_server_node", config_server_node()} }); auto iter = name_to_cap_set.find(cap_set_name); return (iter != name_to_cap_set.end()) ? std::optional<CapabilitySet>(iter->second) : std::nullopt; @@ -65,12 +65,14 @@ CapabilitySet CapabilitySet::content_node() noexcept { CapabilitySet CapabilitySet::container_node() noexcept { return CapabilitySet::of({Capability::content_document_api(), + Capability::container_document_api(), Capability::content_search_api()}) .union_of(shared_app_node_capabilities()); } CapabilitySet CapabilitySet::telemetry() noexcept { return CapabilitySet::of({Capability::content_status_pages(), + Capability::content_state_api(), Capability::content_metrics_api(), Capability::container_state_api(), Capability::metricsproxy_metrics_api(), @@ -88,7 +90,7 @@ CapabilitySet CapabilitySet::logserver_node() noexcept { return shared_app_node_capabilities(); } -CapabilitySet CapabilitySet::config_server() noexcept { +CapabilitySet CapabilitySet::config_server_node() noexcept { return CapabilitySet::of({Capability::client_filereceiver_api(), Capability::container_management_api(), Capability::slobrok_api(), diff --git a/vespalib/src/vespa/vespalib/net/tls/capability_set.h b/vespalib/src/vespa/vespalib/net/tls/capability_set.h index 8aad28a4162..0811739217e 100644 --- a/vespalib/src/vespa/vespalib/net/tls/capability_set.h +++ b/vespalib/src/vespa/vespalib/net/tls/capability_set.h @@ -111,7 +111,7 @@ public: [[nodiscard]] static CapabilitySet telemetry() noexcept; [[nodiscard]] static CapabilitySet cluster_controller_node() noexcept; [[nodiscard]] static CapabilitySet logserver_node() noexcept; - [[nodiscard]] static CapabilitySet config_server() noexcept; + [[nodiscard]] static CapabilitySet config_server_node() noexcept; [[nodiscard]] static CapabilitySet make_with_all_capabilities() noexcept; [[nodiscard]] static constexpr CapabilitySet make_empty() noexcept { return {}; }; diff --git a/vespalib/src/vespa/vespalib/net/tls/policy_checking_certificate_verifier.cpp b/vespalib/src/vespa/vespalib/net/tls/policy_checking_certificate_verifier.cpp index e280434c59f..a3f9b3f52c9 100644 --- a/vespalib/src/vespa/vespalib/net/tls/policy_checking_certificate_verifier.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/policy_checking_certificate_verifier.cpp @@ -74,13 +74,15 @@ VerificationResult PolicyConfiguredCertificateVerifier::verify(const PeerCredent return VerificationResult::make_authorized_with_all_capabilities(); } CapabilitySet caps; + bool matched_any_policy = false; for (const auto& policy : _authorized_peers.peer_policies()) { if (matches_all_policy_requirements(peer_creds, policy)) { caps.add_all(policy.granted_capabilities()); + matched_any_policy = true; } } - if (!caps.empty()) { - return VerificationResult::make_authorized_with_capabilities(std::move(caps)); + if (matched_any_policy) { + return VerificationResult::make_authorized_with_capabilities(caps); } else { return VerificationResult::make_not_authorized(); } diff --git a/vespalib/src/vespa/vespalib/net/tls/verification_result.cpp b/vespalib/src/vespa/vespalib/net/tls/verification_result.cpp index f1e50d3115e..37b95c3c07a 100644 --- a/vespalib/src/vespa/vespalib/net/tls/verification_result.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/verification_result.cpp @@ -6,14 +6,18 @@ namespace vespalib::net::tls { -VerificationResult::VerificationResult() = default; +VerificationResult::VerificationResult() noexcept + : _granted_capabilities(), + _authorized(false) +{} -VerificationResult::VerificationResult(CapabilitySet granted_capabilities) - : _granted_capabilities(std::move(granted_capabilities)) +VerificationResult::VerificationResult(bool authorized, CapabilitySet granted_capabilities) noexcept + : _granted_capabilities(granted_capabilities), + _authorized(authorized) {} -VerificationResult::VerificationResult(const VerificationResult&) = default; -VerificationResult& VerificationResult::operator=(const VerificationResult&) = default; +VerificationResult::VerificationResult(const VerificationResult&) noexcept = default; +VerificationResult& VerificationResult::operator=(const VerificationResult&) noexcept = default; VerificationResult::VerificationResult(VerificationResult&&) noexcept = default; VerificationResult& VerificationResult::operator=(VerificationResult&&) noexcept = default; VerificationResult::~VerificationResult() = default; @@ -29,18 +33,18 @@ void VerificationResult::print(asciistream& os) const { } VerificationResult -VerificationResult::make_authorized_with_capabilities(CapabilitySet granted_capabilities) { - return VerificationResult(std::move(granted_capabilities)); +VerificationResult::make_authorized_with_capabilities(CapabilitySet granted_capabilities) noexcept { + return {true, granted_capabilities}; } VerificationResult -VerificationResult::make_authorized_with_all_capabilities() { - return VerificationResult(CapabilitySet::make_with_all_capabilities()); +VerificationResult::make_authorized_with_all_capabilities() noexcept { + return {true, CapabilitySet::make_with_all_capabilities()}; } VerificationResult -VerificationResult::make_not_authorized() { - return {}; +VerificationResult::make_not_authorized() noexcept { + return {false, CapabilitySet::make_empty()}; } asciistream& operator<<(asciistream& os, const VerificationResult& res) { diff --git a/vespalib/src/vespa/vespalib/net/tls/verification_result.h b/vespalib/src/vespa/vespalib/net/tls/verification_result.h index 92b32ad92f7..896908f7c13 100644 --- a/vespalib/src/vespa/vespalib/net/tls/verification_result.h +++ b/vespalib/src/vespa/vespalib/net/tls/verification_result.h @@ -16,22 +16,27 @@ namespace vespalib::net::tls { * This result contains the union set of all capabilities granted by the matching * authorization rules. If no rules matched, the set will be empty. The capability * set will also be empty for a default-constructed instance. + * + * It is possible for a VerificationResult to be successful but with an empty + * capability set. If capabilities are enforced, this will effectively only + * allow mTLS handshakes to go through, allowing rudimentary health checking. */ class VerificationResult { CapabilitySet _granted_capabilities; + bool _authorized; - explicit VerificationResult(CapabilitySet granted_capabilities); + VerificationResult(bool authorized, CapabilitySet granted_capabilities) noexcept; public: - VerificationResult(); - VerificationResult(const VerificationResult&); - VerificationResult& operator=(const VerificationResult&); + VerificationResult() noexcept; // Unauthorized by default + VerificationResult(const VerificationResult&) noexcept; + VerificationResult& operator=(const VerificationResult&) noexcept; VerificationResult(VerificationResult&&) noexcept; VerificationResult& operator=(VerificationResult&&) noexcept; ~VerificationResult(); - // Returns true iff at least one capability been granted. + // Returns true iff the peer matched at least one policy or authorization is not enforced. [[nodiscard]] bool success() const noexcept { - return !_granted_capabilities.empty(); + return _authorized; } [[nodiscard]] const CapabilitySet& granted_capabilities() const noexcept { @@ -40,9 +45,9 @@ public: void print(asciistream& os) const; - static VerificationResult make_authorized_with_capabilities(CapabilitySet granted_capabilities); - static VerificationResult make_authorized_with_all_capabilities(); - static VerificationResult make_not_authorized(); + static VerificationResult make_authorized_with_capabilities(CapabilitySet granted_capabilities) noexcept; + static VerificationResult make_authorized_with_all_capabilities() noexcept; + static VerificationResult make_not_authorized() noexcept; }; asciistream& operator<<(asciistream&, const VerificationResult&); diff --git a/vespalib/src/vespa/vespalib/net/wakeup_pipe.cpp b/vespalib/src/vespa/vespalib/net/wakeup_pipe.cpp index 60900289e79..b8025bfcf9f 100644 --- a/vespalib/src/vespa/vespalib/net/wakeup_pipe.cpp +++ b/vespalib/src/vespa/vespalib/net/wakeup_pipe.cpp @@ -2,34 +2,40 @@ #include "wakeup_pipe.h" #include "socket_utils.h" +#include <vespa/vespalib/util/require.h> #include <unistd.h> namespace vespalib { WakeupPipe::WakeupPipe() - : _pipe() + : _reader(), + _writer() { - socketutils::nonblocking_pipe(_pipe); + int pipe[2]; + socketutils::nonblocking_pipe(pipe); + _reader.reset(pipe[0]); + _writer.reset(pipe[1]); } -WakeupPipe::~WakeupPipe() -{ - close(_pipe[0]); - close(_pipe[1]); -} +WakeupPipe::~WakeupPipe() = default; void WakeupPipe::write_token() { char token = 'T'; - [[maybe_unused]] ssize_t res = write(_pipe[1], &token, 1); + ssize_t res = _writer.write(&token, 1); + if (res < 0) { + res = -errno; + } + REQUIRE(res > 0 || res == -EAGAIN || res == -EWOULDBLOCK); } void WakeupPipe::read_tokens() { char token_trash[128]; - [[maybe_unused]] ssize_t res = read(_pipe[0], token_trash, sizeof(token_trash)); + ssize_t res = _reader.read(token_trash, sizeof(token_trash)); + REQUIRE(res > 0); } } diff --git a/vespalib/src/vespa/vespalib/net/wakeup_pipe.h b/vespalib/src/vespa/vespalib/net/wakeup_pipe.h index b52f7f9e32d..36c88b205c0 100644 --- a/vespalib/src/vespa/vespalib/net/wakeup_pipe.h +++ b/vespalib/src/vespa/vespalib/net/wakeup_pipe.h @@ -2,6 +2,8 @@ #pragma once +#include "socket_handle.h" + namespace vespalib { //----------------------------------------------------------------------------- @@ -15,11 +17,12 @@ namespace vespalib { **/ class WakeupPipe { private: - int _pipe[2]; + SocketHandle _reader; + SocketHandle _writer; public: WakeupPipe(); ~WakeupPipe(); - int get_read_fd() const { return _pipe[0]; } + int get_read_fd() const { return _reader.get(); } void write_token(); void read_tokens(); }; diff --git a/vespalib/src/vespa/vespalib/portal/http_connection.cpp b/vespalib/src/vespa/vespalib/portal/http_connection.cpp index 6ea56e2659c..3d8edf2fc2e 100644 --- a/vespalib/src/vespa/vespalib/portal/http_connection.cpp +++ b/vespalib/src/vespa/vespalib/portal/http_connection.cpp @@ -245,14 +245,16 @@ HttpConnection::handle_event(bool, bool) } void -HttpConnection::respond_with_content(const vespalib::string &content_type, - const vespalib::string &content) +HttpConnection::respond_with_content(vespalib::stringref content_type, + vespalib::stringref content) { { OutputWriter dst(_output, CHUNK_SIZE); dst.printf("HTTP/1.1 200 OK\r\n"); dst.printf("Connection: close\r\n"); - dst.printf("Content-Type: %s\r\n", content_type.c_str()); + dst.printf("Content-Type: "); + dst.write(content_type.data(), content_type.size()); + dst.printf("\r\n"); dst.printf("Content-Length: %zu\r\n", content.size()); emit_http_security_headers(dst); dst.printf("\r\n"); @@ -263,11 +265,13 @@ HttpConnection::respond_with_content(const vespalib::string &content_type, } void -HttpConnection::respond_with_error(int code, const vespalib::string &msg) +HttpConnection::respond_with_error(int code, vespalib::stringref msg) { { OutputWriter dst(_output, CHUNK_SIZE); - dst.printf("HTTP/1.1 %d %s\r\n", code, msg.c_str()); + dst.printf("HTTP/1.1 %d ", code); + dst.write(msg.data(), msg.size()); + dst.printf("\r\n"); dst.printf("Connection: close\r\n"); dst.printf("\r\n"); } diff --git a/vespalib/src/vespa/vespalib/portal/http_connection.h b/vespalib/src/vespa/vespalib/portal/http_connection.h index 03d23351e7d..8540cb87e1d 100644 --- a/vespalib/src/vespa/vespalib/portal/http_connection.h +++ b/vespalib/src/vespa/vespalib/portal/http_connection.h @@ -53,9 +53,9 @@ public: // Precondition: handshake must have been completed const net::ConnectionAuthContext &auth_context() const noexcept { return *_auth_ctx; } - void respond_with_content(const vespalib::string &content_type, - const vespalib::string &content); - void respond_with_error(int code, const vespalib::string &msg); + void respond_with_content(vespalib::stringref content_type, + vespalib::stringref content); + void respond_with_error(int code, const vespalib::stringref msg); }; } // namespace vespalib::portal diff --git a/vespalib/src/vespa/vespalib/portal/portal.cpp b/vespalib/src/vespa/vespalib/portal/portal.cpp index a98562f6504..691ddaef495 100644 --- a/vespalib/src/vespa/vespalib/portal/portal.cpp +++ b/vespalib/src/vespa/vespalib/portal/portal.cpp @@ -77,8 +77,8 @@ Portal::GetRequest::export_params() const } void -Portal::GetRequest::respond_with_content(const vespalib::string &content_type, - const vespalib::string &content) +Portal::GetRequest::respond_with_content(vespalib::stringref content_type, + vespalib::stringref content) { assert(active()); _conn->respond_with_content(content_type, content); @@ -86,7 +86,7 @@ Portal::GetRequest::respond_with_content(const vespalib::string &content_type, } void -Portal::GetRequest::respond_with_error(int code, const vespalib::string &msg) +Portal::GetRequest::respond_with_error(int code, vespalib::stringref msg) { assert(active()); _conn->respond_with_error(code, msg); diff --git a/vespalib/src/vespa/vespalib/portal/portal.h b/vespalib/src/vespa/vespalib/portal/portal.h index 314dd6e7de9..6954d800c91 100644 --- a/vespalib/src/vespa/vespalib/portal/portal.h +++ b/vespalib/src/vespa/vespalib/portal/portal.h @@ -65,9 +65,9 @@ public: bool has_param(const vespalib::string &name) const; const vespalib::string &get_param(const vespalib::string &name) const; std::map<vespalib::string, vespalib::string> export_params() const; - void respond_with_content(const vespalib::string &content_type, - const vespalib::string &content); - void respond_with_error(int code, const vespalib::string &msg); + void respond_with_content(vespalib::stringref content_type, + vespalib::stringref content); + void respond_with_error(int code, vespalib::stringref msg); const net::ConnectionAuthContext &auth_context() const noexcept; ~GetRequest(); }; diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index c8536fc68c1..663b3b65638 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -63,6 +63,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT programoptions.cpp random.cpp rcuvector.cpp + ref_counted.cpp regexp.cpp require.cpp resource_limits.cpp diff --git a/vespalib/src/vespa/vespalib/util/assert.cpp b/vespalib/src/vespa/vespalib/util/assert.cpp index 4844f6346fe..840310e4673 100644 --- a/vespalib/src/vespa/vespalib/util/assert.cpp +++ b/vespalib/src/vespa/vespalib/util/assert.cpp @@ -4,12 +4,11 @@ #include <vespa/defaults.h> #include <vespa/vespalib/util/backtrace.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/time.h> #include <vespa/vespalib/component/vtag.h> #include <fstream> #include <map> #include <mutex> -#include <chrono> -#include <iomanip> #include <vespa/log/log.h> LOG_SETUP(".vespa.assert"); @@ -63,9 +62,7 @@ assertOnceOrLog(const char *expr, const char *key, size_t freq) LOG(error, "assert(%s) named '%s' failed first time. Stacktrace = %s", expr, key, vespalib::getStackTrace(0).c_str()); std::ofstream assertStream(rememberAssert.c_str()); - std::chrono::time_point now = std::chrono::system_clock::now(); - std::time_t now_c = std::chrono::system_clock::to_time_t(now); - assertStream << std::put_time(std::gmtime(&now_c), "%F %T") << " assert(" << expr + assertStream << to_string(system_clock::now()) << " assert(" << expr << ") named " << key << " failed" << std::endl; assertStream.close(); } diff --git a/vespalib/src/vespa/vespalib/util/overview.h b/vespalib/src/vespa/vespalib/util/overview.h deleted file mode 100644 index 74bf0a33310..00000000000 --- a/vespalib/src/vespa/vespalib/util/overview.h +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/*! \mainpage Vespalib - C++ utility library for Vespa components - * - * \section intro_sec Introduction - * - * vespalib is a collection of simple utility classes shared - * between most Vespa components that are written in C++. - * Most of these aren't Vespa specific in any way, except that - * they reflect the Vespa code standard and conventions. - * - * \section install_sec Installation - * - * install vespa_vespalib_dev - * - * \section overview_sec Overview - * - * Most of the classes in vespalib can be used by themselves, - * for a full list see the alphabetical class list. - * Here are the major groups of classes: - * - * Generation counter - * - * vespalib::GenCnt - * - * Reference counting - * - * <BR> vespalib::ReferenceCounter - * - * Advanced pointer utilities - * - * \ref vespalib::PtrHolder<T> - * - * Simple hashmap - * - * \ref vespalib::HashMap<T> - * - * Scope guards for easier exception-safety - * - * vespalib::CounterGuard - * <BR> vespalib::DirPointer - * <BR> vespalib::FileDescriptor - * <BR> vespalib::FilePointer - * <BR> \ref vespalib::MaxValueGuard<T> - * <BR> \ref vespalib::ValueGuard<T> - * - * General utility macros - * <BR> \ref VESPA_STRLOC - * <BR> \ref VESPA_STRINGIZE(str) - * - * Standalone testing framework - * - * vespalib::TestApp - * - * Text handling - * - * vespalib::Utf8Reader - * <BR> vespalib::Utf8Writer - * <BR> vespalib::LowerCase - */ diff --git a/vespalib/src/vespa/vespalib/util/process_memory_stats.cpp b/vespalib/src/vespa/vespalib/util/process_memory_stats.cpp index f7e8e087727..41f1e282c4b 100644 --- a/vespalib/src/vespa/vespalib/util/process_memory_stats.cpp +++ b/vespalib/src/vespa/vespalib/util/process_memory_stats.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <algorithm> #include <vector> +#include <cinttypes> #include <vespa/log/log.h> diff --git a/vespalib/src/vespa/vespalib/util/ref_counted.cpp b/vespalib/src/vespa/vespalib/util/ref_counted.cpp new file mode 100644 index 00000000000..47bb59bb287 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/ref_counted.cpp @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "ref_counted.h" +#include <cassert> + +namespace vespalib { + +void +enable_ref_counted::internal_addref(uint32_t cnt) const noexcept +{ + // relaxed because: + // the thread obtaining the new reference already has a reference + auto prev = _refs.fetch_add(cnt, std::memory_order_relaxed); + assert(prev > 0); + assert(_guard == MAGIC); +} + +void +enable_ref_counted::internal_subref(uint32_t cnt, [[maybe_unused]] uint32_t reserve) const noexcept +{ + // release because: + // our changes to the object must be visible to the deleter + auto prev = _refs.fetch_sub(cnt, std::memory_order_release); + assert(prev >= (reserve + cnt)); + assert(_guard == MAGIC); + if (prev == cnt) { + // acquire because: + // we need to see all object changes before deleting it + std::atomic_thread_fence(std::memory_order_acquire); + delete this; + } +} + +uint32_t +enable_ref_counted::count_refs() const noexcept { + auto result = _refs.load(std::memory_order_relaxed); + assert(result > 0); + assert(_guard == MAGIC); + return result; +} + +enable_ref_counted::~enable_ref_counted() noexcept +{ + // protect against early/double delete and memory overwrites + assert(_refs.load(std::memory_order_relaxed) == 0); + assert(_guard == MAGIC); + _guard = 0; +} + +} diff --git a/vespalib/src/vespa/vespalib/util/ref_counted.h b/vespalib/src/vespa/vespalib/util/ref_counted.h new file mode 100644 index 00000000000..aff8ae7bb7e --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/ref_counted.h @@ -0,0 +1,120 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <atomic> +#include <concepts> +#include <memory> +#include <utility> + +// This file contains code that implements intrusive reference +// counting with smart handles (ref_counted<T> points to T, T inherits +// enable_ref_counted). + +// Functions with names starting with 'internal' are not intended for +// direct use, but are available for completeness (enables incremental +// re-write of code from bald to smart pointers). + +namespace vespalib { + +// This is the actual reference count. It cannot be moved or copied, +// but it can be changed even when const. Classes that want to be +// counted need to inherit from this. The destructor is virtual to +// make sure the right one is called. This also enables the subref +// function to directly delete the shared object. +class enable_ref_counted +{ + static constexpr uint32_t MAGIC = 0xcc56a933; +private: + uint32_t _guard; + mutable std::atomic<uint32_t> _refs; +protected: + enable_ref_counted() noexcept : _guard(MAGIC), _refs(1) {} +public: + virtual ~enable_ref_counted() noexcept; + enable_ref_counted(enable_ref_counted &&) = delete; + enable_ref_counted(const enable_ref_counted &) = delete; + enable_ref_counted &operator=(enable_ref_counted &&) = delete; + enable_ref_counted &operator=(const enable_ref_counted &) = delete; + void internal_addref(uint32_t cnt) const noexcept; + void internal_addref() const noexcept { internal_addref(1); } + void internal_subref(uint32_t cnt, uint32_t reserve) const noexcept; + void internal_subref() const noexcept { internal_subref(1, 0); } + uint32_t count_refs() const noexcept; +}; + +// This is the handle to a shared object. The handle itself is not +// thread safe. +template <typename T> +class ref_counted +{ + // carefully placed here to give the best error messages + static_assert(std::derived_from<T,enable_ref_counted>); + template <typename X> friend class ref_counted; +private: + T *_ptr; + ref_counted(T *ptr) noexcept : _ptr(ptr) {} + void maybe_subref() noexcept { + if (_ptr) [[likely]] { + _ptr->internal_subref(); + } + } + static T *maybe_addref(T *ptr) noexcept { + if (ptr) [[likely]] { + ptr->internal_addref(); + } + return ptr; + } + void replace_with(T *ptr) noexcept { + maybe_subref(); + _ptr = ptr; + } +public: + ref_counted() noexcept : _ptr(nullptr) {} + ref_counted(ref_counted &&rhs) noexcept : _ptr(std::exchange(rhs._ptr, nullptr)) {} + ref_counted(const ref_counted &rhs) noexcept : _ptr(maybe_addref(rhs._ptr)) {} + ref_counted &operator=(ref_counted &&rhs) noexcept { + replace_with(std::exchange(rhs._ptr, nullptr)); + return *this; + } + ref_counted &operator=(const ref_counted &rhs) noexcept { + replace_with(maybe_addref(rhs._ptr)); + return *this; + } + template <typename X> ref_counted(ref_counted<X> &&rhs) noexcept : _ptr(std::exchange(rhs._ptr, nullptr)) {} + template <typename X> ref_counted(const ref_counted<X> &rhs) noexcept : _ptr(maybe_addref(rhs._ptr)) {} + template <typename X> ref_counted &operator=(ref_counted<X> &&rhs) noexcept { + replace_with(std::exchange(rhs._ptr, nullptr)); + return *this; + } + template <typename X> ref_counted &operator=(const ref_counted<X> &rhs) noexcept { + replace_with(maybe_addref(rhs._ptr)); + return *this; + } + T *operator->() const noexcept { return _ptr; } + T &operator*() const noexcept { return *_ptr; } + operator bool() const noexcept { return (_ptr != nullptr); } + ~ref_counted() noexcept { maybe_subref(); } + // NB: will not call subref + T *internal_detach() noexcept { return std::exchange(_ptr, nullptr); } + // NB: will not call addref + static ref_counted internal_attach(T *ptr) noexcept { return ref_counted(ptr); } +}; + +// similar to make_shared; +// create a reference counted object and return a handle to it +template <typename T, typename ... Args> +ref_counted<T> make_ref_counted(Args && ... args) { + return ref_counted<T>::internal_attach(new T(std::forward<Args>(args)...)); +} + +// similar to shared_from_this; +// create a new handle to a reference counted object +// (NB: object must still be valid) +template <typename T> +ref_counted<T> ref_counted_from(T &t) noexcept { + t.internal_addref(); + return ref_counted<T>::internal_attach(std::addressof(t)); +} + +} diff --git a/vespalib/src/vespa/vespalib/util/referencecounter.h b/vespalib/src/vespa/vespalib/util/referencecounter.h deleted file mode 100644 index 28bc927fe03..00000000000 --- a/vespalib/src/vespa/vespalib/util/referencecounter.h +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @file referencecounter.h - * @author Thomas F. Gundersen - * @version $Id$ - * @date 2004-03-19 - **/ - -#pragma once - -#include <atomic> -#include <cassert> - -namespace vespalib -{ - -/** - * @brief Inherit this class to create a self-destroying class. - * - * Allows for objects to be shared without worrying about who "owns" - * the object. When a new owner is given the object, addRef() should - * be called. When finished with the object, subRef() should be - * called. When the last owner calls subRef(), the object is deleted. -*/ -class ReferenceCounter -{ -public: - /** - * @brief Constructor. The object will initially have 1 reference. - **/ - ReferenceCounter() : _refs(1) {} - - /** - * @brief Add an owner of this object. - * - * When the owner is finished with the - * object, call subRef(). - **/ - void addRef() { _refs.fetch_add(1); } - - /** - * @brief Remove an owner of this object. - * - * If that was the last owner, delete the object. - **/ - void subRef() { - if (_refs.fetch_sub(1) == 1) { - delete this; - } - } - unsigned refCount() const { return _refs; } -protected: - /** - * @brief Destructor. Does sanity check only. - **/ - virtual ~ReferenceCounter() { assert (_refs == 0); }; -private: - std::atomic<unsigned> _refs; -}; - -} // namespace vespalib - diff --git a/vespalib/src/vespa/vespalib/util/thread.cpp b/vespalib/src/vespa/vespalib/util/thread.cpp index 1c8fef53ba3..6113924e352 100644 --- a/vespalib/src/vespa/vespalib/util/thread.cpp +++ b/vespalib/src/vespa/vespalib/util/thread.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "thread.h" +#include <cstring> namespace vespalib::thread { @@ -11,4 +12,11 @@ std::thread start(Runnable &runnable, Runnable::init_fun_t init_fun_in) { }); } +size_t as_zu(std::thread::id id) { + size_t res = 0; + static_assert(sizeof(id) <= sizeof(res)); + std::memcpy(&res, &id, sizeof(id)); + return res; +} + } diff --git a/vespalib/src/vespa/vespalib/util/thread.h b/vespalib/src/vespa/vespalib/util/thread.h index d919b8f2ab7..9f3ebd89165 100644 --- a/vespalib/src/vespa/vespalib/util/thread.h +++ b/vespalib/src/vespa/vespalib/util/thread.h @@ -10,6 +10,7 @@ namespace vespalib { namespace thread { [[nodiscard]] std::thread start(Runnable &runnable, Runnable::init_fun_t init_fun); +size_t as_zu(std::thread::id id); } /** @@ -23,15 +24,18 @@ private: public: ThreadPool() noexcept : _threads() {} void start(Runnable &runnable, Runnable::init_fun_t init_fun) { - _threads.reserve(_threads.size() + 1); + reserve(size() + 1); _threads.push_back(thread::start(runnable, std::move(init_fun))); } template<typename F, typename... Args> requires std::invocable<F,Args...> void start(F &&f, Args && ... args) { - _threads.reserve(_threads.size() + 1); + reserve(size() + 1); _threads.emplace_back(std::forward<F>(f), std::forward<Args>(args)...); }; + void reserve(size_t capacity) { _threads.reserve(capacity); } + size_t size() const { return _threads.size(); } + bool empty() const { return _threads.empty(); } void join() { for (auto &thread: _threads) { thread.join(); diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp index 8b6427d9391..8af14366293 100644 --- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp +++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp @@ -1,29 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "threadstackexecutorbase.h" -#include <vespa/fastos/thread.h> namespace vespalib { -namespace thread { - -struct ThreadInit : public FastOS_Runnable { - Runnable &worker; - ThreadStackExecutorBase::init_fun_t init_fun; - - explicit ThreadInit(Runnable &worker_in, ThreadStackExecutorBase::init_fun_t init_fun_in) - : worker(worker_in), init_fun(std::move(init_fun_in)) {} - - void Run(FastOS_ThreadInterface *, void *) override; -}; - -void -ThreadInit::Run(FastOS_ThreadInterface *, void *) { - init_fun(worker); -} - -} - ThreadStackExecutorBase::Worker::Worker() : lock(), cond(), @@ -155,7 +135,7 @@ ThreadStackExecutorBase::run() ThreadStackExecutorBase::ThreadStackExecutorBase(uint32_t taskLimit, init_fun_t init_fun) : SyncableThreadExecutor(), Runnable(), - _pool(std::make_unique<FastOS_ThreadPool>()), + _pool(), _lock(), _cond(), _stats(), @@ -167,7 +147,7 @@ ThreadStackExecutorBase::ThreadStackExecutorBase(uint32_t taskLimit, init_fun_t _taskCount(0), _taskLimit(taskLimit), _closed(false), - _thread_init(std::make_unique<thread::ThreadInit>(*this, std::move(init_fun))) + _init_fun(init_fun) { assert(taskLimit > 0); } @@ -177,15 +157,13 @@ ThreadStackExecutorBase::start(uint32_t threads) { assert(threads > 0); for (uint32_t i = 0; i < threads; ++i) { - FastOS_ThreadInterface *thread = _pool->NewThread(_thread_init.get()); - assert(thread != nullptr); - (void)thread; + _pool.start(*this, _init_fun); } } size_t ThreadStackExecutorBase::getNumThreads() const { - return _pool->GetNumStartedThreads(); + return _pool.size(); } void @@ -315,12 +293,11 @@ ThreadStackExecutorBase::cleanup() { shutdown().sync(); _executorCompletion.countDown(); - _pool->Close(); + _pool.join(); } ThreadStackExecutorBase::~ThreadStackExecutorBase() { - assert(_pool->isClosed()); assert(_taskCount == 0); assert(_blocked.empty()); } diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h index 501fde92f4c..765499b73bc 100644 --- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h +++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h @@ -2,21 +2,17 @@ #pragma once +#include "thread.h" #include "threadexecutor.h" #include "eventbarrier.hpp" #include "arrayqueue.hpp" #include "gate.h" -#include "runnable.h" #include "executor_idle_tracking.h" #include <vector> #include <functional> -class FastOS_ThreadPool; - namespace vespalib { -namespace thread { struct ThreadInit; } - /** * An executor service that executes tasks in multiple threads. **/ @@ -73,7 +69,7 @@ private: void unblock(); }; - std::unique_ptr<FastOS_ThreadPool> _pool; + ThreadPool _pool; mutable std::mutex _lock; std::condition_variable _cond; ExecutorStats _stats; @@ -86,7 +82,7 @@ private: uint32_t _taskCount; uint32_t _taskLimit; bool _closed; - std::unique_ptr<thread::ThreadInit> _thread_init; + init_fun_t _init_fun; static thread_local ThreadStackExecutorBase *_master; void block_thread(const unique_lock &, BlockedThread &blocked_thread); @@ -225,4 +221,3 @@ public: }; } // namespace vespalib - diff --git a/vespalib/src/vespa/vespalib/util/time.h b/vespalib/src/vespa/vespalib/util/time.h index 27f359071ae..2cb53df8ae2 100644 --- a/vespalib/src/vespa/vespalib/util/time.h +++ b/vespalib/src/vespa/vespalib/util/time.h @@ -43,8 +43,9 @@ constexpr double to_s(duration d) { system_time to_utc(steady_time ts); -constexpr duration from_s(double seconds) { - return std::chrono::duration_cast<duration>(std::chrono::duration<double>(seconds)); +template <typename duration_type = duration> +constexpr duration_type from_s(double seconds) { + return std::chrono::duration_cast<duration_type>(std::chrono::duration<double>(seconds)); } constexpr int64_t count_s(duration d) { diff --git a/vespalog/src/logctl/logctl.cpp b/vespalog/src/logctl/logctl.cpp index 4cf44e9cd22..9a8987fc462 100644 --- a/vespalog/src/logctl/logctl.cpp +++ b/vespalog/src/logctl/logctl.cpp @@ -6,6 +6,7 @@ #include <vespa/log/component.h> #include <optional> +#include <cstring> #include <unistd.h> #include <dirent.h> #include <sys/stat.h> @@ -16,7 +17,7 @@ LOG_SETUP("vespa-logctl"); using namespace ns_log; static void modifyLevels(const char *file, const char *component, const char *levels, - bool shouldCreateFile, bool shouldCreateEntry); + bool shouldCreateFile, bool shouldCreateEntry); static void readLevels(const char *file, const char *component); diff --git a/vespalog/src/logger/llreader.cpp b/vespalog/src/logger/llreader.cpp index cb25b6747bf..200f4bb039d 100644 --- a/vespalog/src/logger/llreader.cpp +++ b/vespalog/src/logger/llreader.cpp @@ -1,17 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <stdlib.h> -#include <string.h> -#include <stdio.h> -#include <fcntl.h> -#include <errno.h> -#include <unistd.h> -#include <sys/time.h> #include "llreader.h" +#include <cstdlib> +#include <cstring> +#include <unistd.h> namespace ns_log { - InputBuf::InputBuf(int fd) : _inputfd(fd), _size(1000), diff --git a/vespalog/src/test/bufferedlogskiptest.cpp b/vespalog/src/test/bufferedlogskiptest.cpp index 8b7f1982678..29e5e119c34 100644 --- a/vespalog/src/test/bufferedlogskiptest.cpp +++ b/vespalog/src/test/bufferedlogskiptest.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/bufferedlogger.h> +#include <vespa/log/internal.h> #include <fstream> #include <iostream> @@ -9,6 +10,17 @@ LOG_SETUP("bufferedlogskiptest"); +using namespace std::literals::chrono_literals; + +/** Test timer returning just a given time. Used in tests to fake time. */ +struct TestTimer : public ns_log::Timer { + uint64_t & _time; + TestTimer(uint64_t & timeVar) : _time(timeVar) { } + ns_log::system_time getTimestamp() const noexcept override { + return ns_log::system_time(std::chrono::microseconds(_time)); + } +}; + std::string readFile(const std::string& file) { std::ostringstream ost; std::ifstream is(file.c_str()); @@ -75,8 +87,8 @@ main(int argc, char **argv) } ns_log::Logger::fakePid = true; uint64_t timer; - ns_log_logger.setTimer(std::unique_ptr<ns_log::Timer>(new ns_log::TestTimer(timer))); - ns_log::BufferedLogger::instance().setTimer(std::unique_ptr<ns_log::Timer>(new ns_log::TestTimer(timer))); + ns_log_logger.setTimer(std::make_unique<TestTimer>(timer)); + ns_log::BufferedLogger::instance().setTimer(std::make_unique<TestTimer>(timer)); reset(timer); testSkipBufferOnDebug(argv[1], timer); diff --git a/vespalog/src/test/bufferedlogtest.cpp b/vespalog/src/test/bufferedlogtest.cpp index 365f8fb85a7..a2dfdd7c6b8 100644 --- a/vespalog/src/test/bufferedlogtest.cpp +++ b/vespalog/src/test/bufferedlogtest.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/bufferedlogger.h> +#include <vespa/log/internal.h> #include "bufferedlogtest.logger1.h" #include "bufferedlogtest.logger2.h" @@ -12,6 +13,18 @@ LOG_SETUP("bufferedlogtest"); +using namespace std::literals::chrono_literals; + + +/** Test timer returning just a given time. Used in tests to fake time. */ +struct TestTimer : public ns_log::Timer { + uint64_t & _time; + TestTimer(uint64_t & timeVar) : _time(timeVar) { } + ns_log::system_time getTimestamp() const noexcept override { + return ns_log::system_time(std::chrono::microseconds(_time)); + } +}; + std::string readFile(const std::string& file) { std::ostringstream ost; std::ifstream is(file.c_str()); @@ -386,8 +399,8 @@ main(int argc, char **argv) ns_log::Logger::fakePid = true; ns_log::BufferedLogger::instance().setMaxCacheSize(10); uint64_t timer; - ns_log_logger.setTimer(std::unique_ptr<ns_log::Timer>(new ns_log::TestTimer(timer))); - ns_log::BufferedLogger::instance().setTimer(std::unique_ptr<ns_log::Timer>(new ns_log::TestTimer(timer))); + ns_log_logger.setTimer(std::make_unique<TestTimer>(timer)); + ns_log::BufferedLogger::instance().setTimer(std::make_unique<TestTimer>(timer)); reset(timer); testThatEntriesWithHighCountIsKept(argv[1], timer); diff --git a/vespalog/src/test/threads/CMakeLists.txt b/vespalog/src/test/threads/CMakeLists.txt index 00de16ff005..19fe2511025 100644 --- a/vespalog/src/test/threads/CMakeLists.txt +++ b/vespalog/src/test/threads/CMakeLists.txt @@ -4,6 +4,5 @@ vespa_add_executable(vespalog_threads_test_app TEST testthreads.cpp DEPENDS vespalog - fastos ) vespa_add_test(NAME vespalog_threads_test_app COMMAND vespalog_threads_test_app vespa.log ENVIRONMENT "VESPA_LOG_TARGET=file:vespa.log") diff --git a/vespalog/src/test/threads/testthreads.cpp b/vespalog/src/test/threads/testthreads.cpp index 0c9b3a1cdb2..6a9d5c18a18 100644 --- a/vespalog/src/test/threads/testthreads.cpp +++ b/vespalog/src/test/threads/testthreads.cpp @@ -9,6 +9,7 @@ #include <unistd.h> #include <sys/stat.h> #include <cstdlib> +#include <cstring> #include <vector> using std::string; diff --git a/vespalog/src/vespa/log/CMakeLists.txt b/vespalog/src/vespa/log/CMakeLists.txt index dbeba17334b..82536b1f4e9 100644 --- a/vespalog/src/vespa/log/CMakeLists.txt +++ b/vespalog/src/vespa/log/CMakeLists.txt @@ -18,3 +18,4 @@ vespa_add_library(vespalog reject-filter.cpp INSTALL lib64 ) +target_link_libraries(vespalog PUBLIC ${CMAKE_THREAD_LIBS_INIT}) diff --git a/vespalog/src/vespa/log/bufferedlogger.cpp b/vespalog/src/vespa/log/bufferedlogger.cpp index 33ff3da7366..7123d1bb5fa 100644 --- a/vespalog/src/vespa/log/bufferedlogger.cpp +++ b/vespalog/src/vespa/log/bufferedlogger.cpp @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "bufferedlogger.h" +#include "internal.h" #include <boost/multi_index_container.hpp> #include <boost/multi_index/identity.hpp> -#include <boost/multi_index/member.hpp> #include <boost/multi_index/mem_fun.hpp> #include <boost/multi_index/ordered_index.hpp> #include <boost/multi_index/sequenced_index.hpp> @@ -14,6 +14,8 @@ #include <cstdarg> #include <mutex> +using namespace std::literals::chrono_literals; + namespace ns_log { // implementation details for BufferedLogger @@ -25,7 +27,7 @@ public: /** Lock needed to access cache. */ mutable std::mutex _mutex; - static uint64_t _countFactor; + static duration _countFactor; /** Struct keeping information about log message. */ struct Entry { @@ -35,7 +37,7 @@ public: std::string _token; std::string _message; uint32_t _count; - uint64_t _timestamp; + system_time _timestamp; Logger* _logger; Entry(const Entry &); @@ -44,13 +46,13 @@ public: Entry & operator=(Entry &&) noexcept; Entry(Logger::LogLevel level, const char* file, int line, const std::string& token, const std::string& message, - uint64_t timestamp, Logger&); + system_time timestamp, Logger&); ~Entry(); bool operator==(const Entry& entry) const; bool operator<(const Entry& entry) const; - uint64_t getAgeFactor() const; + system_time getAgeFactor() const; std::string toString() const; }; @@ -73,7 +75,7 @@ public: >, boost::multi_index::ordered_non_unique< boost::multi_index::const_mem_fun< - Entry, uint64_t, &Entry::getAgeFactor + Entry, system_time, &Entry::getAgeFactor > > > @@ -85,7 +87,7 @@ public: LogCacheBack _cacheBack; uint32_t _maxCacheSize; - uint64_t _maxEntryAge; + duration _maxEntryAge; /** Empty buffer and write all log entries in it. */ void flush(); @@ -97,7 +99,7 @@ public: * Flush parts of cache, so we're below max size and only have messages of * acceptable age. Calling this, _mutex should already be locked. */ - void trimCache(uint64_t currentTime); + void trimCache(system_time currentTime); /** * Trim the cache up to current time. Used externally to check if we @@ -125,11 +127,11 @@ public: }; // Let each hit count for 5 seconds -uint64_t BackingBuffer::_countFactor = VESPA_LOG_COUNTAGEFACTOR * 1000 * 1000; +duration BackingBuffer::_countFactor = VESPA_LOG_COUNTAGEFACTOR * 1s; BackingBuffer::Entry::Entry(Logger::LogLevel level, const char* file, int line, const std::string& token, const std::string& msg, - uint64_t timestamp, Logger& l) + system_time timestamp, Logger& l) : _level(level), _file(file), _line(line), @@ -171,11 +173,11 @@ BackingBuffer::Entry::toString() const std::ostringstream ost; ost << "Entry(" << _level << ", " << _file << ":" << _line << ": " << _message << " [" << _token << "], count " << _count - << ", timestamp " << _timestamp << ")"; + << ", timestamp " << count_us(_timestamp.time_since_epoch()) << ")"; return ost.str(); } -uint64_t +system_time BackingBuffer::Entry::getAgeFactor() const { return _timestamp + _countFactor * _count; @@ -191,9 +193,7 @@ BackingBuffer::BackingBuffer() { } -BackingBuffer::~BackingBuffer() -{ -} +BackingBuffer::~BackingBuffer() = default; BufferedLogger::BufferedLogger() { @@ -206,16 +206,25 @@ BufferedLogger::~BufferedLogger() } namespace { - typedef boost::multi_index::nth_index< - BackingBuffer::LogCacheFront, 0>::type LogCacheFrontTimestamp; - typedef boost::multi_index::nth_index< - BackingBuffer::LogCacheFront, 1>::type LogCacheFrontToken; - typedef boost::multi_index::nth_index< - BackingBuffer::LogCacheBack, 0>::type LogCacheBackTimestamp; - typedef boost::multi_index::nth_index< - BackingBuffer::LogCacheBack, 1>::type LogCacheBackToken; - typedef boost::multi_index::nth_index< - BackingBuffer::LogCacheBack, 2>::type LogCacheBackAge; + +typedef boost::multi_index::nth_index< + BackingBuffer::LogCacheFront, 0>::type LogCacheFrontTimestamp; +typedef boost::multi_index::nth_index< + BackingBuffer::LogCacheFront, 1>::type LogCacheFrontToken; +typedef boost::multi_index::nth_index< + BackingBuffer::LogCacheBack, 0>::type LogCacheBackTimestamp; +typedef boost::multi_index::nth_index< + BackingBuffer::LogCacheBack, 1>::type LogCacheBackToken; +typedef boost::multi_index::nth_index< + BackingBuffer::LogCacheBack, 2>::type LogCacheBackAge; + +struct TimeStampWrapper : public Timer { + TimeStampWrapper(system_time timeStamp) : _timeStamp(timeStamp) {} + system_time getTimestamp() const noexcept override { return _timeStamp; } + + system_time _timeStamp; +}; + } void @@ -258,7 +267,7 @@ BackingBuffer::logImpl(Logger& l, Logger::LogLevel level, _cacheBack.get<1>().replace(it2, copy); } else { // If entry didn't already exist, add it to the cache and log it - l.doLogCore(entry._timestamp, level, file, line, message.c_str(), message.size()); + l.doLogCore(TimeStampWrapper(entry._timestamp), level, file, line, message.c_str(), message.size()); _cacheFront.push_back(entry); } trimCache(entry._timestamp); @@ -268,16 +277,12 @@ void BackingBuffer::flush() { std::lock_guard<std::mutex> guard(_mutex); - for (LogCacheBack::const_iterator it = _cacheBack.begin(); - it != _cacheBack.end(); ++it) - { - log(*it); + for (const auto & entry : _cacheBack) { + log(entry); } _cacheBack.clear(); - for (LogCacheFront::const_iterator it = _cacheFront.begin(); - it != _cacheFront.end(); ++it) - { - log(*it); + for (const auto & entry : _cacheFront) { + log(entry); } _cacheFront.clear(); } @@ -288,7 +293,7 @@ BufferedLogger::flush() { } void -BackingBuffer::trimCache(uint64_t currentTime) +BackingBuffer::trimCache(system_time currentTime) { // Remove entries that have been in here too long. while (!_cacheBack.empty() && @@ -310,9 +315,7 @@ BackingBuffer::trimCache(uint64_t currentTime) _cacheBack.push_back(e); } // Remove entries from back based on count modified age. - for (uint32_t i = _cacheFront.size() + _cacheBack.size(); - i > _maxCacheSize; --i) - { + for (uint32_t i = _cacheFront.size() + _cacheBack.size(); i > _maxCacheSize; --i) { log(*_cacheBack.get<2>().begin()); _cacheBack.get<2>().erase(_cacheBack.get<2>().begin()); } @@ -330,10 +333,10 @@ BackingBuffer::log(const Entry& e) const if (e._count > 1) { std::ostringstream ost; ost << e._message << " (Repeated " << (e._count - 1) - << " times since " << (e._timestamp / 1000000) << "." - << std::setw(6) << std::setfill('0') << (e._timestamp % 1000000) + << " times since " << count_s(e._timestamp.time_since_epoch()) << "." + << std::setw(6) << std::setfill('0') << (count_us(e._timestamp.time_since_epoch()) % 1000000) << ")"; - e._logger->doLogCore(_timer->getTimestamp(), e._level, e._file.c_str(), + e._logger->doLogCore(*_timer, e._level, e._file.c_str(), e._line, ost.str().c_str(), ost.str().size()); } } @@ -344,16 +347,12 @@ BackingBuffer::toString() const std::ostringstream ost; ost << "Front log cache content:\n"; std::lock_guard<std::mutex> guard(_mutex); - for (LogCacheFront::const_iterator it = _cacheFront.begin(); - it != _cacheFront.end(); ++it) - { - ost << " " << it->toString() << "\n"; + for (const auto & entry : _cacheFront) { + ost << " " << entry.toString() << "\n"; } ost << "Back log cache content:\n"; - for (LogCacheBack::const_iterator it = _cacheBack.begin(); - it != _cacheBack.end(); ++it) - { - ost << " " << it->toString() << "\n"; + for (const auto & entry : _cacheBack) { + ost << " " << entry.toString() << "\n"; } return ost.str(); } @@ -366,12 +365,12 @@ BufferedLogger::setMaxCacheSize(uint32_t size) { void BufferedLogger::setMaxEntryAge(uint64_t seconds) { - _backing->_maxEntryAge = seconds * 1000000; + _backing->_maxEntryAge = std::chrono::seconds(seconds); } void -BufferedLogger::setCountFactor(uint64_t factor) { - _backing->_countFactor = factor * 1000000; +BufferedLogger::setCountFactor(uint64_t seconds) { + _backing->_countFactor = std::chrono::seconds(seconds); } /** Set a fake timer to use for log messages. Used in unit testing. */ diff --git a/vespalog/src/vespa/log/bufferedlogger.h b/vespalog/src/vespa/log/bufferedlogger.h index 8baa32445a5..f3aee2fb3f6 100644 --- a/vespalog/src/vespa/log/bufferedlogger.h +++ b/vespalog/src/vespa/log/bufferedlogger.h @@ -167,10 +167,10 @@ class BackingBuffer; class BufferedLogger { BackingBuffer *_backing; - BufferedLogger(const BufferedLogger & buf); - BufferedLogger & operator = (const BufferedLogger & buf); public: + BufferedLogger(const BufferedLogger & buf) = delete; + BufferedLogger & operator = (const BufferedLogger & buf) = delete; BufferedLogger(); ~BufferedLogger(); @@ -179,7 +179,7 @@ public: // of the default settings for applications void setMaxCacheSize(uint32_t size); void setMaxEntryAge(uint64_t seconds); - void setCountFactor(uint64_t factor); + void setCountFactor(uint64_t seconds); void doLog(Logger&, Logger::LogLevel level, const char *file, int line, const std::string& token, @@ -188,9 +188,6 @@ public: /** Empty buffer and write all log entries in it. */ void flush(); - /** Gives all current content of log buffer. Useful for debugging. */ - std::string toString() const; - /** Set a fake timer to use for log messages. Used in unit testing. */ void setTimer(std::unique_ptr<Timer> timer); diff --git a/vespalog/src/vespa/log/component.cpp b/vespalog/src/vespa/log/component.cpp index 009a69ad0c5..36b1d15e457 100644 --- a/vespalog/src/vespa/log/component.cpp +++ b/vespalog/src/vespa/log/component.cpp @@ -9,6 +9,7 @@ LOG_SETUP_INDIRECT(".log.control", "$Id$"); #include "component.h" #include "control-file.h" #include "internal.h" +#include <cstring> namespace ns_log { diff --git a/vespalog/src/vespa/log/internal.h b/vespalog/src/vespa/log/internal.h index c25c7cc44b6..02fd1693775 100644 --- a/vespalog/src/vespa/log/internal.h +++ b/vespalog/src/vespa/log/internal.h @@ -3,6 +3,7 @@ #include <string> #include <cstdlib> +#include <chrono> namespace ns_log { @@ -20,4 +21,24 @@ public: [[nodiscard]] const char *what() const { return _what.c_str(); } }; +using system_time = std::chrono::system_clock::time_point; +using duration = system_time::duration; + +constexpr int64_t +count_s(duration d) noexcept { + return std::chrono::duration_cast<std::chrono::seconds>(d).count(); +} + +constexpr int64_t +count_us(duration d) noexcept { + return std::chrono::duration_cast<std::chrono::microseconds>(d).count(); +} + +// XXX this is way too complicated, must be some simpler way to do this +/** Timer class used to retrieve timestamp, such that we can override in test */ +struct Timer { + virtual ~Timer() = default; + virtual system_time getTimestamp() const noexcept; +}; + } // end namespace ns_log diff --git a/vespalog/src/vespa/log/llparser.cpp b/vespalog/src/vespa/log/llparser.cpp index 1585b9fde33..063220e50ef 100644 --- a/vespalog/src/vespa/log/llparser.cpp +++ b/vespalog/src/vespa/log/llparser.cpp @@ -4,6 +4,7 @@ #include "llparser.h" #include "internal.h" #include <cstdlib> +#include <cstring> #include <unistd.h> #include <sys/time.h> #include <cassert> diff --git a/vespalog/src/vespa/log/log-target-file.cpp b/vespalog/src/vespa/log/log-target-file.cpp index 4337d6b5bbb..87a9810e3c7 100644 --- a/vespalog/src/vespa/log/log-target-file.cpp +++ b/vespalog/src/vespa/log/log-target-file.cpp @@ -1,15 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "log.h" +LOG_SETUP(".log"); +#include "log-target-file.h" +#include "internal.h" + #include <unistd.h> #include <cstring> #include <fcntl.h> #include <cerrno> #include <cassert> -#include "log.h" -LOG_SETUP(".log"); -#include "log-target-file.h" -#include "internal.h" - namespace ns_log { #ifndef O_LARGEFILE diff --git a/vespalog/src/vespa/log/log.cpp b/vespalog/src/vespa/log/log.cpp index 7430815122d..73cbeef08aa 100644 --- a/vespalog/src/vespa/log/log.cpp +++ b/vespalog/src/vespa/log/log.cpp @@ -6,7 +6,6 @@ LOG_SETUP_INDIRECT(".log", "$Id$"); #undef LOG #define LOG LOG_INDIRECT -#include "lock.h" #include "log-target.h" #include "internal.h" #include "control-file.h" @@ -15,18 +14,16 @@ LOG_SETUP_INDIRECT(".log", "$Id$"); #include <vespa/defaults.h> #include <cassert> #include <cstdarg> +#include <cstring> +#include <cinttypes> #include <unistd.h> #include <sys/time.h> namespace ns_log { -uint64_t Timer::getTimestamp() const { - struct timeval tv; - gettimeofday(&tv, nullptr); - uint64_t timestamp = tv.tv_sec; - timestamp *= 1000000; - timestamp += tv.tv_usec; - return timestamp; +system_time +Timer::getTimestamp() const noexcept { + return std::chrono::system_clock::now(); } LogTarget *Logger::_target = 0; @@ -135,7 +132,7 @@ Logger::ensureHostname() Logger::Logger(const char *name, const char *rcsId) : _logLevels(ControlFile::defaultLevels()), - _timer(new Timer()) + _timer(std::make_unique<Timer>()) { _numInstances++; memset(_rcsId, 0, sizeof(_rcsId)); @@ -172,7 +169,6 @@ Logger::Logger(const char *name, const char *rcsId) } } - Logger::~Logger() { _numInstances--; @@ -190,6 +186,10 @@ Logger::~Logger() } } +void +Logger::setTimer(std::unique_ptr<Timer> timer) { + _timer = std::move(timer); +} int Logger::setRcsId(const char *id) @@ -220,8 +220,7 @@ Logger::tryLog(int sizeofPayload, LogLevel level, const char *file, int line, co const int actualSize = vsnprintf(payload, sizeofPayload, fmt, args); if (actualSize < sizeofPayload) { - uint64_t timestamp = _timer->getTimestamp(); - doLogCore(timestamp, level, file, line, payload, actualSize); + doLogCore(*_timer, level, file, line, payload, actualSize); } delete[] payload; return actualSize; @@ -242,9 +241,11 @@ Logger::doLog(LogLevel level, const char *file, int line, const char *fmt, ...) ns_log::BufferedLogger::instance().trimCache(); } -void Logger::doLogCore(uint64_t timestamp, LogLevel level, - const char *file, int line, const char *msg, size_t msgSize) +void +Logger::doLogCore(const Timer & timer, LogLevel level, + const char *file, int line, const char *msg, size_t msgSize) { + system_time timestamp = timer.getTimestamp(); const size_t sizeofEscapedPayload(msgSize*4+1); const size_t sizeofTotalMessage(sizeofEscapedPayload + 1000); auto escapedPayload = std::make_unique<char[]>(sizeofEscapedPayload); @@ -281,23 +282,23 @@ void Logger::doLogCore(uint64_t timestamp, LogLevel level, // found to be too inaccurate. int32_t tid = (fakePid ? -1 : gettid(pthread_self()) % 0xffff); + time_t secs = count_s(timestamp.time_since_epoch()); + uint32_t usecs_part = count_us(timestamp.time_since_epoch()) % 1000000; if (_target->makeHumanReadable()) { - time_t secs = static_cast<time_t>(timestamp / 1000000); struct tm tmbuf; localtime_r(&secs, &tmbuf); char timebuf[100]; strftime(timebuf, 100, "%Y-%m-%d %H:%M:%S", &tmbuf); snprintf(totalMessage.get(), sizeofTotalMessage, "[%s.%06u] %d/%d (%s%s) %s: %s\n", - timebuf, static_cast<unsigned int>(timestamp % 1000000), + timebuf, usecs_part, fakePid ? -1 : getpid(), tid, _prefix, _appendix, levelName(level), msg); } else if (level == debug || level == spam) { snprintf(totalMessage.get(), sizeofTotalMessage, - "%u.%06u\t%s\t%d/%d\t%s\t%s%s\t%s\t%s:%d %s%s\n", - static_cast<unsigned int>(timestamp / 1000000), - static_cast<unsigned int>(timestamp % 1000000), + "%lu.%06u\t%s\t%d/%d\t%s\t%s%s\t%s\t%s:%d %s%s\n", + secs, usecs_part, _hostname, fakePid ? -1 : getpid(), tid, _serviceName, _prefix, _appendix, levelName(level), file, line, @@ -305,9 +306,8 @@ void Logger::doLogCore(uint64_t timestamp, LogLevel level, escapedPayload.get()); } else { snprintf(totalMessage.get(), sizeofTotalMessage, - "%u.%06u\t%s\t%d/%d\t%s\t%s%s\t%s\t%s\n", - static_cast<unsigned int>(timestamp / 1000000), - static_cast<unsigned int>(timestamp % 1000000), + "%lu.%06u\t%s\t%d/%d\t%s\t%s%s\t%s\t%s\n", + secs, usecs_part, _hostname, fakePid ? -1 : getpid(), tid, _serviceName, _prefix, _appendix, levelName(level), escapedPayload.get()); @@ -354,35 +354,20 @@ Logger::doEventStarted(const char *name) void Logger::doEventStopped(const char *name, pid_t pid, int exitCode) { - doLog(event, "", 0, "stopped/1 name=\"%s\" pid=%d exitcode=%d", name, - static_cast<int>(pid), exitCode); -} - -void -Logger::doEventReloading(const char *name) -{ - doLog(event, "", 0, "reloading/1 name=\"%s\"", name); -} - -void -Logger::doEventReloaded(const char *name) -{ - doLog(event, "", 0, "reloaded/1 name=\"%s\"", name); + doLog(event, "", 0, "stopped/1 name=\"%s\" pid=%d exitcode=%d", name, static_cast<int>(pid), exitCode); } void Logger::doEventCrash(const char *name, pid_t pid, int signal) { - doLog(event, "", 0, "crash/1 name=\"%s\" pid=%d signal=\"%s\"", name, pid, - strsignal(signal)); + doLog(event, "", 0, "crash/1 name=\"%s\" pid=%d signal=\"%s\"", name, pid, strsignal(signal)); } void Logger::doEventProgress(const char *name, double value, double total) { if (total > 0) { - doLog(event, "", 0, "progress/1 name=\"%s\" value=%.18g total=%.18g", - name, value, total); + doLog(event, "", 0, "progress/1 name=\"%s\" value=%.18g total=%.18g", name, value, total); } else { doLog(event, "", 0, "progress/1 name=\"%s\" value=%.18g", name, value); } diff --git a/vespalog/src/vespa/log/log.h b/vespalog/src/vespa/log/log.h index 888e24a0f28..857bf4f2b97 100644 --- a/vespalog/src/vespa/log/log.h +++ b/vespalog/src/vespa/log/log.h @@ -2,13 +2,8 @@ #pragma once #include <memory> -#include <cstdint> -#include <sys/types.h> // for pid_t #include <new> // for placement new -#include <cstdlib> // for malloc -#include <cstring> // for memset -#include <cstdarg> // for va_list -#include <cinttypes> +#include <sys/types.h> // for pid_t /** * If this macro is defined, the regular LOG calls will go through the @@ -24,7 +19,7 @@ static ns_log::Logger ns_log_logger(__VA_ARGS__) // NOLINT #define LOG_SETUP_INDIRECT(x, id) \ -static ns_log::Logger *ns_log_indirect_logger=NULL; \ +static ns_log::Logger *ns_log_indirect_logger=nullptr; \ static bool logInitialised = false; \ static const char *logName = x; \ static const char *indirectRcsId = id @@ -34,9 +29,6 @@ static const char *indirectRcsId = id #define LOG_INDIRECT_WOULD_LOG(levelName) \ ns_log_indirect_logger->wants(ns_log::Logger::levelName) -#define LOG_RCSID(x) \ -static int log_dummmy __attribute__((unused)) = ns_log_logger.setRcsId(x) - // Define LOG if not using log buffer. Otherwise log buffer will define them #ifndef VESPA_LOG_USELOGBUFFERFORREGULARLOG #define LOG(level, ...) \ @@ -144,20 +136,7 @@ namespace ns_log { class LogTarget; class ControlFile; - -// XXX this is way too complicated, must be some simpler way to do this -/** Timer class used to retrieve timestamp, such that we can override in test */ -struct Timer { - virtual ~Timer() = default; - virtual uint64_t getTimestamp() const; -}; - -/** Test timer returning just a given time. Used in tests to fake time. */ -struct TestTimer : public Timer { - uint64_t & _time; - TestTimer(uint64_t & timeVar) : _time(timeVar) { } - uint64_t getTimestamp() const override { return _time; } -}; +struct Timer; class Logger { public: @@ -174,9 +153,6 @@ public: static bool fakePid; private: - Logger(const Logger &); - Logger& operator =(const Logger &); - unsigned int *_logLevels; static char _prefix[64]; @@ -206,6 +182,8 @@ private: public: ~Logger(); explicit Logger(const char *name, const char *rcsId = nullptr); + Logger(const Logger &) = delete; + Logger & operator=(const Logger &) = delete; int setRcsId(const char *rcsId); static const char *levelName(LogLevel level); @@ -219,14 +197,12 @@ public: * * @param timestamp Time in microseconds. */ - void doLogCore(uint64_t timestamp, LogLevel level, + void doLogCore(const Timer &, LogLevel level, const char *file, int line, const char *msg, size_t msgSize); void doEventStarting(const char *name); void doEventStopping(const char *name, const char *why); void doEventStarted(const char *name); void doEventStopped(const char *name, pid_t pid, int exitCode); - void doEventReloading(const char *name); - void doEventReloaded(const char *name); void doEventCrash(const char *name, pid_t pid, int signal); void doEventProgress(const char *name, double value, double total = 0); void doEventCount(const char *name, uint64_t value); @@ -234,7 +210,7 @@ public: void doEventState(const char *name, const char *value); // Only for unit testing - void setTimer(std::unique_ptr<Timer> timer) { _timer = std::move(timer); } + void setTimer(std::unique_ptr<Timer> timer); // Only for internal use static LogTarget *getCurrentTarget(); diff --git a/vespamalloc/CMakeLists.txt b/vespamalloc/CMakeLists.txt index af71d8b7d82..a6a33dce7ff 100644 --- a/vespamalloc/CMakeLists.txt +++ b/vespamalloc/CMakeLists.txt @@ -6,7 +6,6 @@ add_definitions(-DPARANOID_LEVEL=0) vespa_define_module( TEST_DEPENDS - fastos vespalib vespalog diff --git a/vespamalloc/src/tests/allocfree/allocfree.cpp b/vespamalloc/src/tests/allocfree/allocfree.cpp index ea6f2105d27..693deb49c55 100644 --- a/vespamalloc/src/tests/allocfree/allocfree.cpp +++ b/vespamalloc/src/tests/allocfree/allocfree.cpp @@ -2,7 +2,6 @@ #include "producerconsumer.h" #include <vespa/vespalib/testkit/testapp.h> #include <map> -#include <thread> #include <vespa/log/log.h> LOG_SETUP("allocfree_test"); @@ -73,7 +72,7 @@ int Test::Main() { } TEST_INIT("allocfree_test"); - FastOS_ThreadPool pool; + vespalib::ThreadPool pool; std::map<int, std::shared_ptr<FreeWorker> > freeWorkers; std::map<int, std::shared_ptr<MallocWorker> > mallocWorkers; @@ -86,22 +85,23 @@ int Test::Main() { mallocFreeWorkers[i] = std::shared_ptr<MallocFreeWorker>(new MallocFreeWorker(200, 16, (i%2) ? true : false)); } - + std::atomic<bool> stop_flag(false); for(std::map<int, std::shared_ptr<FreeWorker> >::iterator it(freeWorkers.begin()), mt(freeWorkers.end()); it != mt; it++) { - ASSERT_TRUE(pool.NewThread(it->second.get(), NULL) != NULL); + it->second->start(pool, stop_flag); } for(std::map<int, std::shared_ptr<MallocWorker> >::iterator it(mallocWorkers.begin()), mt(mallocWorkers.end()); it != mt; it++) { - ASSERT_TRUE(pool.NewThread(it->second.get(), NULL) != NULL); + it->second->start(pool, stop_flag); } for(std::map<int, std::shared_ptr<MallocFreeWorker> >::iterator it(mallocFreeWorkers.begin()), mt(mallocFreeWorkers.end()); it != mt; it++) { - ASSERT_TRUE(pool.NewThread(it->second.get(), NULL) != NULL); + it->second->start(pool, stop_flag); } for (; duration > 0; --duration) { LOG(info, "%d seconds left...", duration); std::this_thread::sleep_for(1s); } - pool.Close(); + stop_flag = true; + pool.join(); size_t numFreeOperations(0); size_t numMallocOperations(0); size_t numSameThreadMallocFreeOperations(0); diff --git a/vespamalloc/src/tests/allocfree/linklist.cpp b/vespamalloc/src/tests/allocfree/linklist.cpp index f3f23d726fd..8ab6eb8de8b 100644 --- a/vespamalloc/src/tests/allocfree/linklist.cpp +++ b/vespamalloc/src/tests/allocfree/linklist.cpp @@ -124,7 +124,7 @@ int Test::Main() { ASSERT_EQUAL(1024ul, sizeof(List)); - FastOS_ThreadPool pool; + vespalib::ThreadPool pool; List::AtomicHeadPtr sharedList(List::HeadPtr(nullptr, 1)); fprintf(stderr, "Start populating list\n"); for (size_t i=0; i < NumBlocks; i++) { @@ -156,18 +156,20 @@ int Test::Main() { LinkInOutAndIn pc1(sharedList, 16, false); LinkInOutAndIn pc2(sharedList, 16, true); - ASSERT_TRUE(pool.NewThread(&c1, nullptr) != nullptr); - ASSERT_TRUE(pool.NewThread(&c2, nullptr) != nullptr); - ASSERT_TRUE(pool.NewThread(&p1, nullptr) != nullptr); - ASSERT_TRUE(pool.NewThread(&p2, nullptr) != nullptr); - ASSERT_TRUE(pool.NewThread(&pc1, nullptr) != nullptr); - ASSERT_TRUE(pool.NewThread(&pc2, nullptr) != nullptr); + std::atomic<bool> stop_flag(false); + c1.start(pool, stop_flag); + c2.start(pool, stop_flag); + p1.start(pool, stop_flag); + p2.start(pool, stop_flag); + pc1.start(pool, stop_flag); + pc2.start(pool, stop_flag); for (; duration > 0; --duration) { LOG(info, "%d seconds left...", duration); std::this_thread::sleep_for(1s); } - pool.Close(); + stop_flag = true; + pool.join(); fprintf(stderr, "Did (%lu + %lu) = %lu linkIn operations\n", c1.operations(), c2.operations(), c1.operations() + c2.operations()); fprintf(stderr, "Did (%lu + %lu) = %lu linkOut operations\n", diff --git a/vespamalloc/src/tests/allocfree/producerconsumer.cpp b/vespamalloc/src/tests/allocfree/producerconsumer.cpp index 7edd1495fde..ab3cfe6dac5 100644 --- a/vespamalloc/src/tests/allocfree/producerconsumer.cpp +++ b/vespamalloc/src/tests/allocfree/producerconsumer.cpp @@ -38,7 +38,7 @@ ProducerConsumer::~ProducerConsumer() } -void Consumer::Run(FastOS_ThreadInterface *, void *) { +void Consumer::run(std::atomic<bool> &) { for (;;) { MemList ml = _queue.dequeue(); if (ml == NULL) { @@ -59,8 +59,8 @@ void Consumer::Run(FastOS_ThreadInterface *, void *) { } } -void Producer::Run(FastOS_ThreadInterface *t, void *) { - while (!t->GetBreakFlag()) { +void Producer::run(std::atomic<bool> &stop_flag) { + while (!stop_flag.load(std::memory_order_relaxed)) { MemList ml = new MemListImpl(); for (uint32_t i = 0; i < _cnt; ++i) { ml->push_back(produce()); @@ -71,8 +71,8 @@ void Producer::Run(FastOS_ThreadInterface *t, void *) { _target.close(); } -void ProducerConsumer::Run(FastOS_ThreadInterface *t, void *) { - while (!t->GetBreakFlag()) { +void ProducerConsumer::run(std::atomic<bool> &stop_flag) { + while (!stop_flag.load(std::memory_order_relaxed)) { MemListImpl ml; for (uint32_t i = 0; i < _cnt; ++i) { ml.push_back(produce()); diff --git a/vespamalloc/src/tests/allocfree/producerconsumer.h b/vespamalloc/src/tests/allocfree/producerconsumer.h index 23fb95facd8..b89e328c628 100644 --- a/vespamalloc/src/tests/allocfree/producerconsumer.h +++ b/vespamalloc/src/tests/allocfree/producerconsumer.h @@ -3,7 +3,8 @@ #include <vector> #include "queue.h" -#include <vespa/fastos/thread.h> +#include <vespa/vespalib/util/thread.h> +#include <atomic> namespace vespalib { @@ -11,7 +12,15 @@ typedef std::vector<void *> MemListImpl; typedef MemListImpl * MemList; typedef vespalib::Queue<MemList> MemQueue; -class Consumer : public FastOS_Runnable { +struct RunWithStopFlag { + virtual void run(std::atomic<bool> &stop_flag) = 0; + void start(vespalib::ThreadPool &pool, std::atomic<bool> &stop_flag) { + pool.start([this,&stop_flag](){run(stop_flag);}); + } + virtual ~RunWithStopFlag() = default; +}; + +class Consumer : public RunWithStopFlag { private: MemQueue _queue; bool _inverse; @@ -22,11 +31,11 @@ public: virtual ~Consumer(); void enqueue(const MemList &mem) { _queue.enqueue(mem); } void close() { _queue.close(); } - void Run(FastOS_ThreadInterface *t, void *) override; + void run(std::atomic<bool> &stop_flag) override; uint64_t operations() const { return _operations; } }; -class Producer : public FastOS_Runnable { +class Producer : public RunWithStopFlag { private: Consumer & _target; uint32_t _cnt; @@ -35,11 +44,11 @@ private: public: Producer(uint32_t cnt, Consumer &target); virtual ~Producer(); - void Run(FastOS_ThreadInterface *t, void *) override; + void run(std::atomic<bool> &stop_flag) override; uint64_t operations() const { return _operations; } }; -class ProducerConsumer : public FastOS_Runnable { +class ProducerConsumer : public RunWithStopFlag { private: uint32_t _cnt; bool _inverse; @@ -50,7 +59,7 @@ private: public: ProducerConsumer(uint32_t cnt, bool inverse); virtual ~ProducerConsumer(); - void Run(FastOS_ThreadInterface *t, void *) override; + void run(std::atomic<bool> &stop_flag) override; uint64_t operationsConsumed() const { return _operationsConsumed; } uint64_t operationsProduced() const { return _operationsProduced; } }; diff --git a/vespamalloc/src/tests/test.cpp b/vespamalloc/src/tests/test.cpp index f413412dff0..b3694a6dc0c 100644 --- a/vespamalloc/src/tests/test.cpp +++ b/vespamalloc/src/tests/test.cpp @@ -1,6 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/thread.h> +#include <vespa/vespalib/util/thread.h> #include <cstdlib> #include <cstdio> @@ -24,31 +24,21 @@ void testdd() free(a); } -class Thread : public FastOS_Runnable -{ -private: - void Run(FastOS_ThreadInterface * ti, void * arg) override; -}; +void thread_run(); int main(int, char *[]) { - FastOS_ThreadPool threadPool; + vespalib::ThreadPool threadPool; printf("Main stack(%p)\n", &threadPool); - Thread context; - FastOS_ThreadInterface * th[4]; - for (size_t i=0; i<sizeof(th)/sizeof(th[0]); i++) { - th[i] = threadPool.NewThread(&context); - } - for (size_t i=0; i<sizeof(th)/sizeof(th[0]); i++) { - th[i]->Join(); - delete th[i]; + for (int i = 0; i < 4; ++i) { + threadPool.start([](){thread_run();}); } - + threadPool.join(); return 0; } -void Thread::Run(FastOS_ThreadInterface *, void *) +void thread_run() { char * a = new char [100]; delete [] a; diff --git a/vespamalloc/src/tests/test1/CMakeLists.txt b/vespamalloc/src/tests/test1/CMakeLists.txt index dd7c92a4dac..f3ce9f70f45 100644 --- a/vespamalloc/src/tests/test1/CMakeLists.txt +++ b/vespamalloc/src/tests/test1/CMakeLists.txt @@ -8,12 +8,15 @@ vespa_add_executable(vespamalloc_testatomic_app TEST vespamalloc_util EXTERNAL_DEPENDS ${VESPA_ATOMIC_LIB} + dl ) vespa_add_test(NAME vespamalloc_testatomic_app NO_VALGRIND COMMAND vespamalloc_testatomic_app) vespa_add_executable(vespamalloc_new_test_app TEST SOURCES new_test.cpp + EXTERNAL_DEPENDS + dl ) vespa_add_test(NAME vespamalloc_new_test_app NO_VALGRIND COMMAND vespamalloc_new_test_app) diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java index d5207c6fab8..8df37d1f6ce 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java @@ -11,9 +11,9 @@ import java.util.List; import java.util.logging.Level; /** - * Implementation of a Barrier that handles the case where more than number of members can call synchronize. If - * the number of members that synchronize exceed the expected number, the other members are immediately allowed - * to pass through the barrier. + * Implementation of a Barrier that handles the case where more than number of members can call synchronize. + * Will wait for some time for all servers to do the operation, but will accept the majority of servers to have + * done the operation if it takes longer than a specified amount of time. * * @author Vegard Havdal * @author Ulf Lilleengen @@ -21,16 +21,20 @@ import java.util.logging.Level; class CuratorCompletionWaiter implements Curator.CompletionWaiter { private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(CuratorCompletionWaiter.class.getName()); + private static final Duration waitForAllDefault = Duration.ofSeconds(1); // Make this configurable? + private final Curator curator; private final String barrierPath; private final String myId; private final Clock clock; + private final Duration waitForAll; - CuratorCompletionWaiter(Curator curator, String barrierPath, String myId, Clock clock) { + CuratorCompletionWaiter(Curator curator, String barrierPath, String myId, Clock clock, Duration waitForAll) { this.myId = barrierPath + "/" + myId; this.curator = curator; this.barrierPath = barrierPath; this.clock = clock; + this.waitForAll = waitForAll; } @Override @@ -55,6 +59,7 @@ class CuratorCompletionWaiter implements Curator.CompletionWaiter { private List<String> awaitInternal(Duration timeout) throws Exception { Instant startTime = clock.instant(); Instant endTime = startTime.plus(timeout); + Instant gotQuorumTime = Instant.EPOCH; List<String> respondents = new ArrayList<>(); do { @@ -65,15 +70,20 @@ class CuratorCompletionWaiter implements Curator.CompletionWaiter { respondents + ", all participants: " + curator.zooKeeperEnsembleConnectionSpec()); } - // First, check if all config servers responded + // If all config servers responded, return if (respondents.size() == curator.zooKeeperEnsembleCount()) { - log.log(Level.FINE, () -> barrierCompletedMessage(respondents, startTime)); + logBarrierCompleted(respondents, startTime); break; } - // If some are missing, quorum is enough + // If some are missing, quorum is enough, but wait for all up to ´waitForAll´ seconds before returning if (respondents.size() >= barrierMemberCount()) { - log.log(Level.FINE, () -> barrierCompletedMessage(respondents, startTime)); - break; + if (gotQuorumTime.isBefore(startTime)) + gotQuorumTime = clock.instant(); + + if (Duration.between(clock.instant(), gotQuorumTime.plus(waitForAll)).isNegative()) { + logBarrierCompleted(respondents, startTime); + break; + } } Thread.sleep(100); @@ -82,9 +92,15 @@ class CuratorCompletionWaiter implements Curator.CompletionWaiter { return respondents; } - private String barrierCompletedMessage(List<String> respondents, Instant startTime) { - return barrierPath + " completed in " + Duration.between(startTime, Instant.now()).toString() + - ", " + respondents.size() + "/" + curator.zooKeeperEnsembleCount() + " responded: " + respondents; + private void logBarrierCompleted(List<String> respondents, Instant startTime) { + Duration duration = Duration.between(startTime, Instant.now()); + Level level = duration.minus(Duration.ofSeconds(5)).isNegative() ? Level.FINE : Level.INFO; + log.log(level, () -> barrierCompletedMessage(respondents, duration)); + } + + private String barrierCompletedMessage(List<String> respondents, Duration duration) { + return barrierPath + " completed in " + duration.toString() + + ", " + respondents.size() + "/" + curator.zooKeeperEnsembleCount() + " responded: " + respondents; } @Override @@ -106,10 +122,18 @@ class CuratorCompletionWaiter implements Curator.CompletionWaiter { } public static Curator.CompletionWaiter create(Curator curator, Path barrierPath, String id) { - return new CuratorCompletionWaiter(curator, barrierPath.getAbsolute(), id, Clock.systemUTC()); + return create(curator, barrierPath, id, waitForAllDefault); + } + + public static Curator.CompletionWaiter create(Curator curator, Path barrierPath, String id, Duration waitForAll) { + return new CuratorCompletionWaiter(curator, barrierPath.getAbsolute(), id, Clock.systemUTC(), waitForAll); } public static Curator.CompletionWaiter createAndInitialize(Curator curator, Path parentPath, String waiterNode, String id) { + return createAndInitialize(curator, parentPath, waiterNode, id, waitForAllDefault); + } + + public static Curator.CompletionWaiter createAndInitialize(Curator curator, Path parentPath, String waiterNode, String id, Duration waitForAll) { Path waiterPath = parentPath.append(waiterNode); String debugMessage = log.isLoggable(Level.FINE) ? "Recreating ZK path " + waiterPath : null; @@ -120,7 +144,7 @@ class CuratorCompletionWaiter implements Curator.CompletionWaiter { if (debugMessage != null) log.fine(debugMessage + ": Done"); - return new CuratorCompletionWaiter(curator, waiterPath.getAbsolute(), id, Clock.systemUTC()); + return new CuratorCompletionWaiter(curator, waiterPath.getAbsolute(), id, Clock.systemUTC(), waitForAll); } private int barrierMemberCount() { diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java index c8566015ea1..e5810763bf2 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java @@ -84,13 +84,10 @@ import org.apache.curator.retry.RetryForever; import org.apache.curator.utils.EnsurePath; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.KeeperException.BadVersionException; -import org.apache.zookeeper.KeeperException.NoNodeException; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Stat; import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; - import java.nio.file.Paths; import java.time.Duration; import java.util.ArrayList; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/api/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/api/package-info.java index ef2f1732a5e..4d3f0b1f916 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/api/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/api/package-info.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 5, minor = 3, micro = 0)) +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 0)) package org.apache.curator.framework.api; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/api/transaction/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/api/transaction/package-info.java index 33e58865082..b7423c886e0 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/api/transaction/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/api/transaction/package-info.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 5, minor = 3, micro = 0)) +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 0)) package org.apache.curator.framework.api.transaction; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/listen/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/listen/package-info.java index acffcc0998c..0b2d01ef6ee 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/listen/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/listen/package-info.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 5, minor = 3, micro = 0)) +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 0)) package org.apache.curator.framework.listen; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/package-info.java index ab7fac0e20a..fb5974dd91a 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/package-info.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 5, minor = 3, micro = 0)) +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 0)) package org.apache.curator.framework; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/recipes/atomic/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/recipes/atomic/package-info.java index 50c95084655..b845dbc21a7 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/recipes/atomic/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/recipes/atomic/package-info.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 5, minor = 3, micro = 0)) +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 0)) package org.apache.curator.framework.recipes.atomic; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/recipes/barriers/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/recipes/barriers/package-info.java index 7c0613fa468..76e2920f9ff 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/recipes/barriers/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/recipes/barriers/package-info.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 5, minor = 3, micro = 0)) +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 0)) package org.apache.curator.framework.recipes.barriers; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/recipes/cache/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/recipes/cache/package-info.java index 1728e179e5e..effe94cba89 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/recipes/cache/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/recipes/cache/package-info.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 5, minor = 3, micro = 0)) +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 0)) package org.apache.curator.framework.recipes.cache; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/recipes/locks/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/recipes/locks/package-info.java index 737ec2b2978..7ea656f7112 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/recipes/locks/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/recipes/locks/package-info.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 5, minor = 3, micro = 0)) +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 0)) package org.apache.curator.framework.recipes.locks; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/state/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/state/package-info.java index 010d2b1afdd..667faaa679e 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/state/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/state/package-info.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 5, minor = 3, micro = 0)) +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 0)) package org.apache.curator.framework.state; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/package-info.java b/zkfacade/src/main/java/org/apache/curator/package-info.java index fa007a89313..0d5c8b08327 100644 --- a/zkfacade/src/main/java/org/apache/curator/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/package-info.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 5, minor = 3, micro = 0)) +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 0)) package org.apache.curator; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/retry/package-info.java b/zkfacade/src/main/java/org/apache/curator/retry/package-info.java index 05128a3acbf..07ac5c7c4c5 100644 --- a/zkfacade/src/main/java/org/apache/curator/retry/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/retry/package-info.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 5, minor = 3, micro = 0)) +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 0)) package org.apache.curator.retry; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; |