diff options
98 files changed, 632 insertions, 3131 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index ce11196725f..c995007663d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,7 +68,6 @@ include_directories(BEFORE ${CMAKE_BINARY_DIR}/configdefinitions/src) add_subdirectory(airlift-zstd) add_subdirectory(ann_benchmark) add_subdirectory(application-model) -add_subdirectory(athenz-identity-provider-service) add_subdirectory(client) add_subdirectory(cloud-tenant-cd) add_subdirectory(clustercontroller-apps) diff --git a/athenz-identity-provider-service/CMakeLists.txt b/athenz-identity-provider-service/CMakeLists.txt deleted file mode 100644 index 75208c49bd3..00000000000 --- a/athenz-identity-provider-service/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_jar(athenz-identity-provider-service-jar-with-dependencies.jar) diff --git a/athenz-identity-provider-service/OWNERS b/athenz-identity-provider-service/OWNERS deleted file mode 100644 index 569bf1cc3a1..00000000000 --- a/athenz-identity-provider-service/OWNERS +++ /dev/null @@ -1 +0,0 @@ -bjorncs diff --git a/athenz-identity-provider-service/README.md b/athenz-identity-provider-service/README.md deleted file mode 100644 index b25eb009e1b..00000000000 --- a/athenz-identity-provider-service/README.md +++ /dev/null @@ -1,5 +0,0 @@ -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -# Athenz Identity Provider Service - -An [Athenz Copper Argos](https://github.com/yahoo/athenz/blob/master/docs/copper_argos.md) provider implementation for configserver. - diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml deleted file mode 100644 index f4daa43b8e3..00000000000 --- a/athenz-identity-provider-service/pom.xml +++ /dev/null @@ -1,186 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 - http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <artifactId>athenz-identity-provider-service</artifactId> - <packaging>container-plugin</packaging> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> - <version>8-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> - </parent> - <dependencies> - <!-- PROVIDED --> - <dependency> - <groupId>com.google.inject</groupId> - <artifactId>guice</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>component</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-apache-http-client-bundle</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-dev</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>jdisc_core</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>node-repository</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-provisioning</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-model-api</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespa-athenz</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>security-utils</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - - <!-- TEST --> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>zkfacade</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.junit.jupiter</groupId> - <artifactId>junit-jupiter-api</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.junit.jupiter</groupId> - <artifactId>junit-jupiter-engine</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>orchestrator</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>application</artifactId> - <version>${project.version}</version> - <scope>test</scope> - <exclusions> - <exclusion> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>testutil</artifactId> - <version>${project.version}</version> - <scope>test</scope> - <exclusions> - <exclusion> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - </exclusion> - <exclusion> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>flags</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-test</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - </dependencies> - - <build> - <plugins> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugin</artifactId> - <extensions>true</extensions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <!-- Illegal reflective access by guice. TODO: try to remove for guice >3.0 --> - <argLine> - --add-opens=java.base/java.lang=ALL-UNNAMED - </argLine> - </configuration> - </plugin> - </plugins> - </build> - -</project> diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CertificateExpiryMetricUpdater.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CertificateExpiryMetricUpdater.java deleted file mode 100644 index f3568caac04..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CertificateExpiryMetricUpdater.java +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.component.AbstractComponent; -import com.yahoo.jdisc.Metric; - -import java.time.Duration; -import java.time.Instant; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * @author freva - */ -public class CertificateExpiryMetricUpdater extends AbstractComponent { - - private static final Duration METRIC_REFRESH_PERIOD = Duration.ofMinutes(5); - private static final String ATHENZ_CONFIGSERVER_CERT_METRIC_NAME = "athenz-configserver-cert.expiry.seconds"; - - private final Logger logger = Logger.getLogger(CertificateExpiryMetricUpdater.class.getName()); - private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); - private final Metric metric; - private final ConfigserverSslContextFactoryProvider provider; - - @Inject - public CertificateExpiryMetricUpdater(Metric metric, - ConfigserverSslContextFactoryProvider provider) { - this.metric = metric; - this.provider = provider; - - scheduler.scheduleAtFixedRate(this::updateMetrics, - 30/*initial delay*/, - METRIC_REFRESH_PERIOD.getSeconds(), - TimeUnit.SECONDS); - } - - @Override - public void deconstruct() { - try { - scheduler.shutdownNow(); - scheduler.awaitTermination(30, TimeUnit.SECONDS); - } catch (InterruptedException e) { - throw new RuntimeException("Failed to shutdown certificate expiry metrics updater on time", e); - } - } - - private void updateMetrics() { - try { - Duration keyStoreExpiry = Duration.between(Instant.now(), provider.getCertificateNotAfter()); - metric.set(ATHENZ_CONFIGSERVER_CERT_METRIC_NAME, keyStoreExpiry.getSeconds(), null); - } catch (Exception e) { - logger.log(Level.WARNING, "Failed to update key store expiry metric: " + e.getMessage(), e); - } - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CkmsKeyProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CkmsKeyProvider.java deleted file mode 100644 index c659c454420..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CkmsKeyProvider.java +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.config.provision.Zone; -import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.security.KeyUtils; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.KeyProvider; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; - -import java.security.KeyPair; -import java.security.PrivateKey; -import java.security.PublicKey; -import java.util.HashMap; -import java.util.Map; - -/** - * @author mortent - * @author bjorncs - */ -@SuppressWarnings("unused") // Injected component -public class CkmsKeyProvider implements KeyProvider { - - private final SecretStore secretStore; - private final String secretName; - private final Map<Integer, KeyPair> secrets; - - @Inject - public CkmsKeyProvider(SecretStore secretStore, - Zone zone, - AthenzProviderServiceConfig config) { - this.secretStore = secretStore; - this.secretName = config.secretName(); - this.secrets = new HashMap<>(); - } - - @Override - public PrivateKey getPrivateKey(int version) { - return getKeyPair(version).getPrivate(); - } - - @Override - public PublicKey getPublicKey(int version) { - return getKeyPair(version).getPublic(); - } - - @Override - public KeyPair getKeyPair(int version) { - synchronized (secrets) { - KeyPair keyPair = secrets.get(version); - if (keyPair == null) { - keyPair = readKeyPair(version); - secrets.put(version, keyPair); - } - return keyPair; - } - } - - private KeyPair readKeyPair(int version) { - PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(secretStore.getSecret(secretName, version)); - PublicKey publicKey = KeyUtils.extractPublicKey(privateKey); - return new KeyPair(publicKey, privateKey); - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java deleted file mode 100644 index 61a4a0fe41f..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.jdisc.http.ssl.impl.TlsContextBasedProvider; -import com.yahoo.security.KeyStoreBuilder; -import com.yahoo.security.KeyStoreType; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.SslContextBuilder; -import com.yahoo.security.tls.DefaultTlsContext; -import com.yahoo.security.MutableX509KeyManager; -import com.yahoo.security.tls.PeerAuthentication; -import com.yahoo.security.tls.TlsContext; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; -import com.yahoo.vespa.athenz.client.zts.Identity; -import com.yahoo.vespa.athenz.client.zts.ZtsClient; -import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; -import com.yahoo.vespa.athenz.utils.SiaUtils; -import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; - -import javax.net.ssl.SSLContext; -import java.net.URI; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.security.KeyPair; -import java.security.KeyStore; -import java.security.PrivateKey; -import java.security.PublicKey; -import java.security.cert.X509Certificate; -import java.time.Duration; -import java.time.Instant; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Configures the JDisc https connector with the configserver's Athenz provider certificate and private key. - * - * @author bjorncs - */ -public class ConfigserverSslContextFactoryProvider extends TlsContextBasedProvider { - - private static final String CERTIFICATE_ALIAS = "athenz"; - private static final Duration EXPIRATION_MARGIN = Duration.ofHours(6); - private static final Path VESPA_SIA_DIRECTORY = Paths.get(Defaults.getDefaults().underVespaHome("var/vespa/sia")); - - private static final Logger log = Logger.getLogger(ConfigserverSslContextFactoryProvider.class.getName()); - - private final TlsContext tlsContext; - private final MutableX509KeyManager keyManager = new MutableX509KeyManager(); - private final ScheduledExecutorService scheduler = - Executors.newSingleThreadScheduledExecutor(runnable -> new Thread(runnable, "configserver-ssl-context-factory-provider")); - private final ZtsClient ztsClient; - private final KeyProvider keyProvider; - private final AthenzProviderServiceConfig athenzProviderServiceConfig; - private final AthenzService configserverIdentity; - - @Inject - public ConfigserverSslContextFactoryProvider(ServiceIdentityProvider bootstrapIdentity, - KeyProvider keyProvider, - AthenzProviderServiceConfig config) { - this.athenzProviderServiceConfig = config; - this.ztsClient = new DefaultZtsClient.Builder(URI.create(athenzProviderServiceConfig.ztsUrl())) - .withIdentityProvider(bootstrapIdentity).build(); - this.keyProvider = keyProvider; - this.configserverIdentity = new AthenzService(athenzProviderServiceConfig.domain(), athenzProviderServiceConfig.serviceName()); - - Duration updatePeriod = Duration.ofDays(config.updatePeriodDays()); - Path trustStoreFile = Paths.get(config.athenzCaTrustStore()); - this.tlsContext = createTlsContext(keyProvider, keyManager, trustStoreFile, updatePeriod, configserverIdentity, ztsClient, athenzProviderServiceConfig); - scheduler.scheduleAtFixedRate(new KeystoreUpdater(keyManager), - updatePeriod.toDays()/*initial delay*/, - updatePeriod.toDays(), - TimeUnit.DAYS); - } - - @Override - protected TlsContext getTlsContext(String containerId, int port) { - return tlsContext; - } - - Instant getCertificateNotAfter() { - return keyManager.currentManager().getCertificateChain(CERTIFICATE_ALIAS)[0].getNotAfter().toInstant(); - } - - @Override - public void deconstruct() { - try { - scheduler.shutdownNow(); - scheduler.awaitTermination(30, TimeUnit.SECONDS); - ztsClient.close(); - super.deconstruct(); - } catch (InterruptedException e) { - throw new RuntimeException("Failed to shutdown Athenz certificate updater on time", e); - } - } - - private static TlsContext createTlsContext(KeyProvider keyProvider, - MutableX509KeyManager keyManager, - Path trustStoreFile, - Duration updatePeriod, - AthenzService configserverIdentity, - ZtsClient ztsClient, - AthenzProviderServiceConfig zoneConfig) { - KeyStore keyStore = - tryReadKeystoreFile(configserverIdentity, updatePeriod) - .orElseGet(() -> updateKeystore(configserverIdentity, generateKeystorePassword(), keyProvider, ztsClient, zoneConfig)); - keyManager.updateKeystore(keyStore, new char[0]); - SSLContext sslContext = new SslContextBuilder() - .withTrustStore(trustStoreFile, KeyStoreType.JKS) - .withKeyManager(keyManager) - .build(); - return new DefaultTlsContext(sslContext, PeerAuthentication.WANT); - } - - private static Optional<KeyStore> tryReadKeystoreFile(AthenzService configserverIdentity, Duration updatePeriod) { - Optional<X509Certificate> certificate = SiaUtils.readCertificateFile(VESPA_SIA_DIRECTORY, configserverIdentity); - if (!certificate.isPresent()) return Optional.empty(); - Optional<PrivateKey> privateKey = SiaUtils.readPrivateKeyFile(VESPA_SIA_DIRECTORY, configserverIdentity); - if (!privateKey.isPresent()) return Optional.empty(); - Instant minimumExpiration = Instant.now().plus(updatePeriod).plus(EXPIRATION_MARGIN); - boolean isExpired = certificate.get().getNotAfter().toInstant().isBefore(minimumExpiration); - if (isExpired) return Optional.empty(); - KeyStore keyStore = KeyStoreBuilder.withType(KeyStoreType.JKS) - .withKeyEntry(CERTIFICATE_ALIAS, privateKey.get(), certificate.get()) - .build(); - return Optional.of(keyStore); - } - - private static KeyStore updateKeystore(AthenzService configserverIdentity, - char[] keystorePwd, - KeyProvider keyProvider, - ZtsClient ztsClient, - AthenzProviderServiceConfig zoneConfig) { - PrivateKey privateKey = keyProvider.getPrivateKey(zoneConfig.secretVersion()); - PublicKey publicKey = KeyUtils.extractPublicKey(privateKey); - Identity serviceIdentity = ztsClient.getServiceIdentity(configserverIdentity, - Integer.toString(zoneConfig.secretVersion()), - new KeyPair(publicKey, privateKey), - zoneConfig.certDnsSuffix()); - X509Certificate certificate = serviceIdentity.certificate(); - SiaUtils.writeCertificateFile(VESPA_SIA_DIRECTORY, configserverIdentity, certificate); - SiaUtils.writePrivateKeyFile(VESPA_SIA_DIRECTORY, configserverIdentity, privateKey); - Instant expirationTime = certificate.getNotAfter().toInstant(); - Duration expiry = Duration.between(certificate.getNotBefore().toInstant(), expirationTime); - log.log(Level.INFO, String.format("Got Athenz x509 certificate with expiry %s (expires %s)", expiry, expirationTime)); - return KeyStoreBuilder.withType(KeyStoreType.JKS) - .withKeyEntry(CERTIFICATE_ALIAS, privateKey, keystorePwd, certificate) - .build(); - } - - private static char[] generateKeystorePassword() { - return UUID.randomUUID().toString().toCharArray(); - } - - private class KeystoreUpdater implements Runnable { - final MutableX509KeyManager keyManager; - - KeystoreUpdater(MutableX509KeyManager keyManager) { - this.keyManager = keyManager; - } - - @Override - public void run() { - try { - log.log(Level.INFO, "Updating configserver provider certificate from ZTS"); - char[] keystorePwd = generateKeystorePassword(); - KeyStore keyStore = updateKeystore(configserverIdentity, keystorePwd, keyProvider, ztsClient, athenzProviderServiceConfig); - keyManager.updateKeystore(keyStore, keystorePwd); - log.log(Level.INFO, "Certificate successfully updated"); - } catch (Throwable t) { - log.log(Level.SEVERE, "Failed to update certificate from ZTS: " + t.getMessage(), t); - } - } - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGenerator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGenerator.java deleted file mode 100644 index 5143a38b2c1..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGenerator.java +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.config.provision.Zone; -import com.yahoo.net.HostName; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.identityprovider.api.ClusterType; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; -import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import com.yahoo.vespa.athenz.identityprovider.client.IdentityDocumentSigner; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.Allocation; - -import java.security.PrivateKey; -import java.time.Instant; -import java.util.HashSet; -import java.util.Set; - -/** - * Generates a signed identity document for a given hostname and type - * - * @author mortent - * @author bjorncs - */ -public class IdentityDocumentGenerator { - - private final IdentityDocumentSigner signer = new IdentityDocumentSigner(); - private final NodeRepository nodeRepository; - private final Zone zone; - private final KeyProvider keyProvider; - private final AthenzProviderServiceConfig athenzProviderServiceConfig; - - @Inject - public IdentityDocumentGenerator(AthenzProviderServiceConfig config, - NodeRepository nodeRepository, - Zone zone, - KeyProvider keyProvider) { - this.athenzProviderServiceConfig = config; - this.nodeRepository = nodeRepository; - this.zone = zone; - this.keyProvider = keyProvider; - } - - public SignedIdentityDocument generateSignedIdentityDocument(String hostname, IdentityType identityType) { - try { - Node node = nodeRepository.nodes().node(hostname).orElseThrow(() -> new RuntimeException("Unable to find node " + hostname)); - Allocation allocation = node.allocation().orElseThrow(() -> new RuntimeException("No allocation for node " + node.hostname())); - VespaUniqueInstanceId providerUniqueId = new VespaUniqueInstanceId( - allocation.membership().index(), - allocation.membership().cluster().id().value(), - allocation.owner().instance().value(), - allocation.owner().application().value(), - allocation.owner().tenant().value(), - zone.region().value(), - zone.environment().value(), - identityType); - - Set<String> ips = new HashSet<>(node.ipConfig().primary()); - - PrivateKey privateKey = keyProvider.getPrivateKey(athenzProviderServiceConfig.secretVersion()); - AthenzService providerService = new AthenzService(athenzProviderServiceConfig.domain(), athenzProviderServiceConfig.serviceName()); - - String configServerHostname = HostName.getLocalhost(); - Instant createdAt = Instant.now(); - var clusterType = ClusterType.from(allocation.membership().cluster().type().name()); - String signature = signer.generateSignature( - providerUniqueId, providerService, configServerHostname, - node.hostname(), createdAt, ips, identityType, privateKey); - return new SignedIdentityDocument( - signature, athenzProviderServiceConfig.secretVersion(), providerUniqueId, providerService, - SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION, configServerHostname, node.hostname(), - createdAt, ips, identityType, clusterType); - } catch (Exception e) { - throw new RuntimeException("Exception generating identity document: " + e.getMessage(), e); - } - } - -} - diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityProviderRequestHandler.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityProviderRequestHandler.java deleted file mode 100644 index c1dd70d7656..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityProviderRequestHandler.java +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.yahoo.component.annotation.Inject; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.restapi.RestApi; -import com.yahoo.restapi.RestApiException; -import com.yahoo.restapi.RestApiRequestHandler; -import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; -import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; - -import java.util.logging.Level; - -/** - * Handler implementing the Athenz Identity Provider API (Copper Argos). - * - * @author bjorncs - */ -public class IdentityProviderRequestHandler extends RestApiRequestHandler<IdentityProviderRequestHandler> { - - private final IdentityDocumentGenerator documentGenerator; - private final InstanceValidator instanceValidator; - - @Inject - public IdentityProviderRequestHandler(ThreadedHttpRequestHandler.Context context, - IdentityDocumentGenerator documentGenerator, - InstanceValidator instanceValidator) { - super(context, IdentityProviderRequestHandler::createRestApi); - this.documentGenerator = documentGenerator; - this.instanceValidator = instanceValidator; - } - - private static RestApi createRestApi(IdentityProviderRequestHandler self) { - return RestApi.builder() - .addRoute(RestApi.route("/athenz/v1/provider/identity-document/node/{host}") - .get(self::getNodeIdentityDocument)) - .addRoute(RestApi.route("/athenz/v1/provider/identity-document/tenant/{host}") - .get(self::getTenantIdentityDocument)) - .addRoute(RestApi.route("/athenz/v1/provider/instance") - .post(InstanceConfirmation.class, self::confirmInstance)) - .addRoute(RestApi.route("/athenz/v1/provider/refresh") - .post(InstanceConfirmation.class, self::confirmInstanceRefresh)) - .registerJacksonRequestEntity(InstanceConfirmation.class) - .registerJacksonResponseEntity(InstanceConfirmation.class) - .registerJacksonResponseEntity(SignedIdentityDocumentEntity.class) - // Overriding object mapper to change serialization of timestamps - .setObjectMapper(new ObjectMapper() - .registerModule(new JavaTimeModule()) - .registerModule(new Jdk8Module()) - .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true)) - .build(); - } - - private SignedIdentityDocumentEntity getNodeIdentityDocument(RestApi.RequestContext context) { - String host = context.pathParameters().getString("host").orElse(null); - return getIdentityDocument(host, IdentityType.NODE); - } - - private SignedIdentityDocumentEntity getTenantIdentityDocument(RestApi.RequestContext context) { - String host = context.pathParameters().getString("host").orElse(null); - return getIdentityDocument(host, IdentityType.TENANT); - } - - private InstanceConfirmation confirmInstance(RestApi.RequestContext context, InstanceConfirmation instanceConfirmation) { - log.log(Level.FINE, () -> instanceConfirmation.toString()); - if (!instanceValidator.isValidInstance(instanceConfirmation)) { - log.log(Level.SEVERE, "Invalid instance: " + instanceConfirmation); - throw new RestApiException.Forbidden("Instance is invalid"); - } - return instanceConfirmation; - } - - private InstanceConfirmation confirmInstanceRefresh(RestApi.RequestContext context, InstanceConfirmation instanceConfirmation) { - log.log(Level.FINE, () -> instanceConfirmation.toString()); - if (!instanceValidator.isValidRefresh(instanceConfirmation)) { - log.log(Level.SEVERE, "Invalid instance refresh: " + instanceConfirmation); - throw new RestApiException.Forbidden("Instance is invalid"); - } - return instanceConfirmation; - } - - private SignedIdentityDocumentEntity getIdentityDocument(String hostname, IdentityType identityType) { - if (hostname == null) { - throw new RestApiException.BadRequest("The 'hostname' query parameter is missing"); - } - try { - return EntityBindingsMapper.toSignedIdentityDocumentEntity(documentGenerator.generateSignedIdentityDocument(hostname, identityType)); - } catch (Exception e) { - String message = String.format("Unable to generate identity document for '%s': %s", hostname, e.getMessage()); - log.log(Level.SEVERE, message, e); - throw new RestApiException.InternalServerError(message, e); - } - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceConfirmation.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceConfirmation.java deleted file mode 100644 index 6c09a35ee3d..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceConfirmation.java +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonUnwrapped; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; - -/** - * InstanceConfirmation object as per Athenz InstanceConfirmation API. - * - * @author bjorncs - */ -public class InstanceConfirmation { - - @JsonProperty("provider") public final String provider; - @JsonProperty("domain") public final String domain; - @JsonProperty("service") public final String service; - - @JsonProperty("attestationData") @JsonSerialize(using = SignedIdentitySerializer.class) - public final SignedIdentityDocumentEntity signedIdentityDocument; - @JsonUnwrapped public final Map<String, String> attributes = new HashMap<>(); // optional attributes that Athenz may provide - - @JsonCreator - public InstanceConfirmation(@JsonProperty("provider") String provider, - @JsonProperty("domain") String domain, - @JsonProperty("service") String service, - @JsonProperty("attestationData") @JsonDeserialize(using = SignedIdentityDeserializer.class) - SignedIdentityDocumentEntity signedIdentityDocument) { - this.provider = provider; - this.domain = domain; - this.service = service; - this.signedIdentityDocument = signedIdentityDocument; - } - - @JsonAnySetter - public void set(String name, String value) { - attributes.put(name, value); - } - - @Override - public String toString() { - return "InstanceConfirmation{" + - "provider='" + provider + '\'' + - ", domain='" + domain + '\'' + - ", service='" + service + '\'' + - ", signedIdentityDocument='" + signedIdentityDocument + '\'' + - ", attributes=" + attributes + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - InstanceConfirmation that = (InstanceConfirmation) o; - return Objects.equals(provider, that.provider) && - Objects.equals(domain, that.domain) && - Objects.equals(service, that.service) && - Objects.equals(signedIdentityDocument, that.signedIdentityDocument) && - Objects.equals(attributes, that.attributes); - } - - @Override - public int hashCode() { - return Objects.hash(provider, domain, service, signedIdentityDocument, attributes); - } - - public static class SignedIdentityDeserializer extends JsonDeserializer<SignedIdentityDocumentEntity> { - @Override - public SignedIdentityDocumentEntity deserialize( - JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { - String value = jsonParser.getValueAsString(); - return Utils.getMapper().readValue(value, SignedIdentityDocumentEntity.class); - } - } - - public static class SignedIdentitySerializer extends JsonSerializer<SignedIdentityDocumentEntity> { - @Override - public void serialize( - SignedIdentityDocumentEntity document, JsonGenerator gen, SerializerProvider serializers) throws IOException { - gen.writeString(Utils.getMapper().writeValueAsString(document)); - } - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidator.java deleted file mode 100644 index d8bbf743d8c..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidator.java +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.google.common.net.InetAddresses; -import com.yahoo.component.annotation.Inject; -import com.yahoo.config.model.api.ApplicationInfo; -import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.config.model.api.SuperModelProvider; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.identityprovider.api.ClusterType; -import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; -import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import com.yahoo.vespa.athenz.identityprovider.client.IdentityDocumentSigner; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; - -import java.net.InetAddress; -import java.net.URI; -import java.security.PublicKey; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.function.Supplier; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Verifies that the instance's identity document is valid - * - * @author bjorncs - * @author mortent - */ -public class InstanceValidator { - - private static final Logger log = Logger.getLogger(InstanceValidator.class.getName()); - static final String SERVICE_PROPERTIES_DOMAIN_KEY = "identity.domain"; - static final String SERVICE_PROPERTIES_SERVICE_KEY = "identity.service"; - static final String INSTANCE_ID_DELIMITER = ".instanceid.athenz."; - - public static final String SAN_IPS_ATTRNAME = "sanIP"; - public static final String SAN_DNS_ATTRNAME = "sanDNS"; - public static final String SAN_URI_ATTRNAME = "sanURI"; - - private final AthenzService tenantDockerContainerIdentity; - private final IdentityDocumentSigner signer; - private final KeyProvider keyProvider; - private final SuperModelProvider superModelProvider; - private final NodeRepository nodeRepository; - - @Inject - public InstanceValidator(KeyProvider keyProvider, - SuperModelProvider superModelProvider, - NodeRepository nodeRepository, - AthenzProviderServiceConfig config) { - this(keyProvider, superModelProvider, nodeRepository, new IdentityDocumentSigner(), new AthenzService(config.tenantService())); - } - - public InstanceValidator(KeyProvider keyProvider, - SuperModelProvider superModelProvider, - NodeRepository nodeRepository, - IdentityDocumentSigner identityDocumentSigner, - AthenzService tenantIdentity){ - this.keyProvider = keyProvider; - this.superModelProvider = superModelProvider; - this.nodeRepository = nodeRepository; - this.signer = identityDocumentSigner; - this.tenantDockerContainerIdentity = tenantIdentity; - } - - public boolean isValidInstance(InstanceConfirmation instanceConfirmation) { - try { - validateInstance(instanceConfirmation); - return true; - } catch (ValidationException e) { - log.log(e.logLevel(), e.messageSupplier()); - return false; - } - } - - public void validateInstance(InstanceConfirmation req) throws ValidationException { - SignedIdentityDocument signedIdentityDocument = EntityBindingsMapper.toSignedIdentityDocument(req.signedIdentityDocument); - VespaUniqueInstanceId providerUniqueId = signedIdentityDocument.providerUniqueId(); - ApplicationId applicationId = ApplicationId.from( - providerUniqueId.tenant(), providerUniqueId.application(), providerUniqueId.instance()); - - VespaUniqueInstanceId csrProviderUniqueId = getVespaUniqueInstanceId(req); - if(! providerUniqueId.equals(csrProviderUniqueId)) { - var msg = String.format("Instance %s has invalid provider unique ID in CSR (%s)", providerUniqueId, csrProviderUniqueId); - throw new ValidationException(Level.WARNING, () -> msg); - } - - if (! isSameIdentityAsInServicesXml(applicationId, req.domain, req.service)) { - Supplier<String> msg = () -> "Invalid identity '%s.%s' in services.xml".formatted(req.domain, req.service); - throw new ValidationException(Level.FINE, msg); - } - - log.log(Level.FINE, () -> String.format("Validating instance %s.", providerUniqueId)); - - PublicKey publicKey = keyProvider.getPublicKey(signedIdentityDocument.signingKeyVersion()); - if (! signer.hasValidSignature(signedIdentityDocument, publicKey)) { - var msg = String.format("Instance %s has invalid signature.", providerUniqueId); - throw new ValidationException(Level.SEVERE, () -> msg); - } - - validateAttributes(req, providerUniqueId); - log.log(Level.FINE, () -> String.format("Instance %s is valid.", providerUniqueId)); - } - - // TODO Add actual validation. Cannot reuse isValidInstance as identity document is not part of the refresh request. - // We'll have to perform some validation on the instance id and other fields of the attribute map. - // Separate between tenant and node certificate as well. - public boolean isValidRefresh(InstanceConfirmation confirmation) { - log.log(Level.FINE, () -> String.format("Accepting refresh for instance with identity '%s', provider '%s', instanceId '%s'.", - new AthenzService(confirmation.domain, confirmation.service).getFullName(), - confirmation.provider, - confirmation.attributes.get(SAN_DNS_ATTRNAME))); - try { - validateAttributes(confirmation, getVespaUniqueInstanceId(confirmation)); - return true; - } catch (ValidationException e) { - log.log(e.logLevel(), e.messageSupplier()); - return false; - } catch (Exception e) { - log.log(Level.WARNING, "Encountered exception while refreshing certificate for confirmation: " + confirmation, e); - return false; - } - } - - private VespaUniqueInstanceId getVespaUniqueInstanceId(InstanceConfirmation instanceConfirmation) { - // Find a list of SAN DNS - List<String> sanDNS = Optional.ofNullable(instanceConfirmation.attributes.get(SAN_DNS_ATTRNAME)) - .map(s -> s.split(",")) - .map(Arrays::asList).stream().flatMap(Collection::stream).toList(); - - return sanDNS.stream() - .filter(dns -> dns.contains(INSTANCE_ID_DELIMITER)) - .findFirst() - .map(s -> s.replaceAll(INSTANCE_ID_DELIMITER + ".*", "")) - .map(VespaUniqueInstanceId::fromDottedString) - .orElse(null); - } - - private void validateAttributes(InstanceConfirmation confirmation, VespaUniqueInstanceId vespaUniqueInstanceId) - throws ValidationException { - if(vespaUniqueInstanceId == null) { - var msg = "Unable to find unique instance ID in refresh request: " + confirmation.toString(); - throw new ValidationException(Level.WARNING, () -> msg); - } - - // Find node matching vespa unique id - Node node = nodeRepository.nodes().list().stream() - .filter(n -> n.allocation().isPresent()) - .filter(n -> nodeMatchesVespaUniqueId(n, vespaUniqueInstanceId)) - .findFirst() // Should be only one - .orElse(null); - if(node == null) { - var msg = "Invalid InstanceConfirmation, No nodes matching uniqueId: " + vespaUniqueInstanceId; - throw new ValidationException(Level.WARNING, () -> msg); - } - - // Find list of ipaddresses - List<InetAddress> ips = Optional.ofNullable(confirmation.attributes.get(SAN_IPS_ATTRNAME)) - .map(s -> s.split(",")) - .map(Arrays::asList).stream().flatMap(Collection::stream) - .map(InetAddresses::forString) - .toList(); - - List<InetAddress> nodeIpAddresses = node.ipConfig().primary().stream() - .map(InetAddresses::forString) - .toList(); - - // Validate that ipaddresses in request are valid for node - - if(! nodeIpAddresses.containsAll(ips)) { - var msg = "Invalid InstanceConfirmation, wrong ip in : " + vespaUniqueInstanceId; - throw new ValidationException(Level.WARNING, () -> msg); - } - - var urisCommaSeparated = confirmation.attributes.get(SAN_URI_ATTRNAME); - Set<URI> requestedUris; - try { - requestedUris = Optional.ofNullable(urisCommaSeparated).stream() - .flatMap(s -> Arrays.stream(s.split(","))).map(URI::create).collect(Collectors.toSet()); - } catch (IllegalArgumentException e) { - throw new ValidationException(Level.WARNING, () -> "Invalid SAN URIs: " + urisCommaSeparated, e); - } - var clusterType = node.allocation().map(a -> a.membership().cluster().type()).orElse(null); - Set<URI> allowedUris = clusterType != null - ? Set.of(ClusterType.from(clusterType.name()).asCertificateSanUri()) : Set.of(); - if (!allowedUris.containsAll(requestedUris)) { - Supplier<String> msg = () -> "Illegal SAN URIs: expected '%s' found '%s'".formatted(allowedUris, requestedUris); - throw new ValidationException(Level.WARNING, msg); - } - } - - private boolean nodeMatchesVespaUniqueId(Node node, VespaUniqueInstanceId vespaUniqueInstanceId) { - return node.allocation().map(allocation -> - allocation.membership().index() == vespaUniqueInstanceId.clusterIndex() && - allocation.membership().cluster().id().value().equals(vespaUniqueInstanceId.clusterId()) && - allocation.owner().instance().value().equals(vespaUniqueInstanceId.instance()) && - allocation.owner().application().value().equals(vespaUniqueInstanceId.application()) && - allocation.owner().tenant().value().equals(vespaUniqueInstanceId.tenant())) - .orElse(false); - } - - // If/when we don't care about logging exactly whats wrong, this can be simplified - // TODO Use identity type to determine if this check should be performed - private boolean isSameIdentityAsInServicesXml(ApplicationId applicationId, String domain, String service) { - - Optional<ApplicationInfo> applicationInfo = superModelProvider.getSuperModel().getApplicationInfo(applicationId); - - if (applicationInfo.isEmpty()) { - log.info(String.format("Could not find application info for %s, existing applications: %s", - applicationId.serializedForm(), - superModelProvider.getSuperModel().getAllApplicationInfos())); - return false; - } - - if (tenantDockerContainerIdentity.equals(new AthenzService(domain, service))) { - return true; - } - - Optional<ServiceInfo> matchingServiceInfo = applicationInfo.get() - .getModel() - .getHosts() - .stream() - .flatMap(hostInfo -> hostInfo.getServices().stream()) - .filter(serviceInfo -> serviceInfo.getProperty(SERVICE_PROPERTIES_DOMAIN_KEY).isPresent()) - .filter(serviceInfo -> serviceInfo.getProperty(SERVICE_PROPERTIES_SERVICE_KEY).isPresent()) - .findFirst(); - - if (matchingServiceInfo.isEmpty()) { - log.info(String.format("Application %s has not specified domain/service", applicationId.serializedForm())); - return false; - } - - String domainInConfig = matchingServiceInfo.get().getProperty(SERVICE_PROPERTIES_DOMAIN_KEY).get(); - String serviceInConfig = matchingServiceInfo.get().getProperty(SERVICE_PROPERTIES_SERVICE_KEY).get(); - if (!domainInConfig.equals(domain) || !serviceInConfig.equals(service)) { - log.warning(String.format("domain '%s' or service '%s' does not match the one in config for application %s", - domain, service, applicationId.serializedForm())); - return false; - } - - return true; - } - - public static class ValidationException extends Exception { - private final Level logLevel; - private final Supplier<String> msg; - - public ValidationException(Level logLevel, Supplier<String> msg) { this(logLevel, msg, null); } - public ValidationException(Level logLevel, Supplier<String> msg, Throwable cause) { super(cause); this.logLevel = logLevel; this.msg = msg; } - - @Override public String getMessage() { return msg.get(); } - public Level logLevel() { return logLevel; } - public Supplier<String> messageSupplier() { return msg; } - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/KeyProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/KeyProvider.java deleted file mode 100644 index 324f927fd73..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/KeyProvider.java +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import java.security.KeyPair; -import java.security.PrivateKey; -import java.security.PublicKey; - -/** - * @author bjorncs - */ -public interface KeyProvider { - PrivateKey getPrivateKey(int version); - - PublicKey getPublicKey(int version); - - default KeyPair getKeyPair(int version) { - return new KeyPair(getPublicKey(version), getPrivateKey(version)); - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/Utils.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/Utils.java deleted file mode 100644 index 5c4942f37cb..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/Utils.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; - -/** - * @author bjorncs - */ -public class Utils { - - private static final ObjectMapper mapper = createObjectMapper(); - - public static ObjectMapper getMapper() { - return mapper; - } - - private static ObjectMapper createObjectMapper() { - ObjectMapper mapper = new ObjectMapper(); - mapper.registerModule(new JavaTimeModule()); - return mapper; - } - -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/package-info.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/package-info.java deleted file mode 100644 index 0cb5c9d4f82..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author bjorncs - */ -@ExportPackage -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/Certificates.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/Certificates.java deleted file mode 100644 index df904bf8010..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/Certificates.java +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca; - -import com.yahoo.security.Pkcs10Csr; -import com.yahoo.security.SubjectAlternativeName; -import com.yahoo.security.X509CertificateBuilder; -import com.yahoo.security.X509CertificateUtils; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; - -import java.security.PrivateKey; -import java.security.cert.X509Certificate; -import java.time.Clock; -import java.time.Duration; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.stream.Collectors; - -import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; -import static com.yahoo.security.SubjectAlternativeName.Type.DNS; - -/** - * Helper class for creating {@link X509Certificate}s. - * - * @author mpolden - */ -public class Certificates { - - private static final Duration CERTIFICATE_TTL = Duration.ofDays(30); - private static final String INSTANCE_ID_DELIMITER = ".instanceid.athenz."; - - private final Clock clock; - - public Certificates(Clock clock) { - this.clock = Objects.requireNonNull(clock, "clock must be non-null"); - } - - /** Create a new certificate from csr signed by the given CA private key */ - public X509Certificate create(Pkcs10Csr csr, X509Certificate caCertificate, PrivateKey caPrivateKey) { - var x500principal = caCertificate.getSubjectX500Principal(); - var now = clock.instant(); - var notBefore = now.minus(Duration.ofHours(1)); - var notAfter = now.plus(CERTIFICATE_TTL); - var builder = X509CertificateBuilder.fromCsr(csr, - x500principal, - notBefore, - notAfter, - caPrivateKey, - SHA256_WITH_ECDSA, - X509CertificateBuilder.generateRandomSerialNumber()); - for (var san : csr.getSubjectAlternativeNames()) { - builder = builder.addSubjectAlternativeName(san.decode()); - } - return builder.build(); - } - - /** Returns instance ID parsed from the Subject Alternative Names in given csr */ - public static String instanceIdFrom(Pkcs10Csr csr) { - return getInstanceIdFromSAN(csr.getSubjectAlternativeNames()) - .orElseThrow(() -> new IllegalArgumentException("No instance ID found in CSR")); - } - - public static Optional<String> instanceIdFrom(X509Certificate certificate) { - return getInstanceIdFromSAN(X509CertificateUtils.getSubjectAlternativeNames(certificate)); - } - - private static Optional<String> getInstanceIdFromSAN(List<SubjectAlternativeName> subjectAlternativeNames) { - return subjectAlternativeNames.stream() - .filter(san -> san.getType() == DNS) - .map(SubjectAlternativeName::getValue) - .map(Certificates::parseInstanceId) - .flatMap(Optional::stream) - .map(VespaUniqueInstanceId::asDottedString) - .findFirst(); - } - - private static Optional<VespaUniqueInstanceId> parseInstanceId(String dnsName) { - var delimiterStart = dnsName.indexOf(INSTANCE_ID_DELIMITER); - if (delimiterStart == -1) return Optional.empty(); - dnsName = dnsName.substring(0, delimiterStart); - try { - return Optional.of(VespaUniqueInstanceId.fromDottedString(dnsName)); - } catch (IllegalArgumentException e) { - return Optional.empty(); - } - } - - public static String getSubjectAlternativeNames(Pkcs10Csr csr, SubjectAlternativeName.Type sanType) { - return csr.getSubjectAlternativeNames().stream() - .map(SubjectAlternativeName::decode) - .filter(san -> san.getType() == sanType) - .map(SubjectAlternativeName::getValue) - .collect(Collectors.joining(",")); - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceIdentity.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceIdentity.java deleted file mode 100644 index f33ec4fbd6d..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceIdentity.java +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.instance; - -import java.security.cert.X509Certificate; -import java.util.Objects; -import java.util.Optional; - -/** - * A signed instance identity object that includes a client certificate. This is the result of a successful - * {@link InstanceRegistration} and is the same type as InstanceIdentity in the ZTS API. - * - * @author mpolden - */ -public class InstanceIdentity { - - private final String provider; - private final String service; - private final String instanceId; - private final Optional<X509Certificate> x509Certificate; - - public InstanceIdentity(String provider, String service, String instanceId, Optional<X509Certificate> x509Certificate) { - this.provider = Objects.requireNonNull(provider, "provider must be non-null"); - this.service = Objects.requireNonNull(service, "service must be non-null"); - this.instanceId = Objects.requireNonNull(instanceId, "instanceId must be non-null"); - this.x509Certificate = Objects.requireNonNull(x509Certificate, "x509Certificate must be non-null"); - } - - /** Same as {@link InstanceRegistration#domain()} */ - public String provider() { - return provider; - } - - /** Same as {@link InstanceRegistration#service()} ()} */ - public String service() { - return service; - } - - /** A unique identifier of the instance to which the certificate is issued */ - public String instanceId() { - return instanceId; - } - - /** The issued certificate */ - public Optional<X509Certificate> x509Certificate() { - return x509Certificate; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - InstanceIdentity that = (InstanceIdentity) o; - return provider.equals(that.provider) && - service.equals(that.service) && - instanceId.equals(that.instanceId) && - x509Certificate.equals(that.x509Certificate); - } - - @Override - public int hashCode() { - return Objects.hash(provider, service, instanceId, x509Certificate); - } - -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRefresh.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRefresh.java deleted file mode 100644 index d63ee7f979f..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRefresh.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.instance; - -import com.yahoo.security.Pkcs10Csr; - -import java.util.Objects; - -/** - * Information for refreshing a instance in the system. This is the same type as InstanceRefreshInformation type in - * the ZTS API. - * - * @author mpolden - */ -public class InstanceRefresh { - - private final Pkcs10Csr csr; - - public InstanceRefresh(Pkcs10Csr csr) { - this.csr = Objects.requireNonNull(csr, "csr must be non-null"); - } - - /** The Certificate Signed Request describing the wanted certificate */ - public Pkcs10Csr csr() { - return csr; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - InstanceRefresh that = (InstanceRefresh) o; - return csr.equals(that.csr); - } - - @Override - public int hashCode() { - return Objects.hash(csr); - } - -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRegistration.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRegistration.java deleted file mode 100644 index 231954976bf..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRegistration.java +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.instance; - -import com.yahoo.security.Pkcs10Csr; -import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; - -import java.util.Objects; - -/** - * Information for registering a new instance in the system. This is the same type as InstanceRegisterInformation type - * in the ZTS API. - * - * @author mpolden - */ -public class InstanceRegistration { - - private final String provider; - private final String domain; - private final String service; - private final SignedIdentityDocument attestationData; - private final Pkcs10Csr csr; - - public InstanceRegistration(String provider, String domain, String service, SignedIdentityDocument attestationData, Pkcs10Csr csr) { - this.provider = Objects.requireNonNull(provider, "provider must be non-null"); - this.domain = Objects.requireNonNull(domain, "domain must be non-null"); - this.service = Objects.requireNonNull(service, "service must be non-null"); - this.attestationData = Objects.requireNonNull(attestationData, "attestationData must be non-null"); - this.csr = Objects.requireNonNull(csr, "csr must be non-null"); - } - - /** The provider which issued the attestation data contained in this */ - public String provider() { - return provider; - } - - /** Athenz domain of the instance */ - public String domain() { - return domain; - } - - /** Athenz service of the instance */ - public String service() { - return service; - } - - /** Host document describing this instance (received from config server) */ - public SignedIdentityDocument attestationData() { - return attestationData; - } - - /** The Certificate Signed Request describing the wanted certificate */ - public Pkcs10Csr csr() { - return csr; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - InstanceRegistration that = (InstanceRegistration) o; - return provider.equals(that.provider) && - domain.equals(that.domain) && - service.equals(that.service) && - attestationData.equals(that.attestationData) && - csr.equals(that.csr); - } - - @Override - public int hashCode() { - return Objects.hash(provider, domain, service, attestationData, csr); - } - - @Override - public String toString() { - return "InstanceRegistration{" + - "provider='" + provider + '\'' + - ", domain='" + domain + '\'' + - ", service='" + service + '\'' + - ", attestationData='" + attestationData.toString() + '\'' + - ", csr=" + csr.toString() + - '}'; - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java deleted file mode 100644 index 531a815922b..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.restapi; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.jdisc.http.server.jetty.RequestUtils; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.Path; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.SubjectAlternativeName; -import com.yahoo.security.X509CertificateUtils; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceConfirmation; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceValidator; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; -import com.yahoo.vespa.hosted.ca.Certificates; -import com.yahoo.vespa.hosted.ca.instance.InstanceIdentity; -import com.yahoo.vespa.hosted.ca.instance.InstanceRefresh; -import com.yahoo.yolean.Exceptions; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.security.PrivateKey; -import java.security.cert.X509Certificate; -import java.time.Clock; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.function.Function; -import java.util.logging.Level; - -/** - * REST API for issuing and refreshing node certificates in a hosted Vespa system. - * - * The API implements the following subset of methods from the Athenz ZTS REST API: - * - * - Instance registration - * - Instance refresh - * - * @author mpolden - */ -public class CertificateAuthorityApiHandler extends ThreadedHttpRequestHandler { - - private final SecretStore secretStore; - private final Certificates certificates; - private final String caPrivateKeySecretName; - private final String caCertificateSecretName; - private final InstanceValidator instanceValidator; - - @Inject - public CertificateAuthorityApiHandler(Context ctx, SecretStore secretStore, AthenzProviderServiceConfig athenzProviderServiceConfig, InstanceValidator instanceValidator) { - this(ctx, secretStore, new Certificates(Clock.systemUTC()), athenzProviderServiceConfig, instanceValidator); - } - - CertificateAuthorityApiHandler(Context ctx, SecretStore secretStore, Certificates certificates, AthenzProviderServiceConfig athenzProviderServiceConfig, InstanceValidator instanceValidator) { - super(ctx); - this.secretStore = secretStore; - this.certificates = certificates; - this.caPrivateKeySecretName = athenzProviderServiceConfig.secretName(); - this.caCertificateSecretName = athenzProviderServiceConfig.caCertSecretName(); - this.instanceValidator = instanceValidator; - } - - @Override - public HttpResponse handle(HttpRequest request) { - try { - switch (request.getMethod()) { - case POST: return handlePost(request); - default: return ErrorResponse.methodNotAllowed("Method " + request.getMethod() + " is unsupported"); - } - } catch (IllegalArgumentException e) { - return ErrorResponse.badRequest(request.getMethod() + " " + request.getUri() + " failed: " + Exceptions.toMessageString(e)); - } catch (RuntimeException e) { - log.log(Level.WARNING, "Unexpected error handling " + request.getMethod() + " " + request.getUri(), e); - return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); - } - } - - private HttpResponse handlePost(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/ca/v1/instance/")) return registerInstance(request); - if (path.matches("/ca/v1/instance/{provider}/{domain}/{service}/{instanceId}")) return refreshInstance(request, path.get("provider"), path.get("service"), path.get("instanceId")); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse registerInstance(HttpRequest request) { - var instanceRegistration = deserializeRequest(request, InstanceSerializer::registrationFromSlime); - - InstanceConfirmation confirmation = new InstanceConfirmation(instanceRegistration.provider(), instanceRegistration.domain(), instanceRegistration.service(), EntityBindingsMapper.toSignedIdentityDocumentEntity(instanceRegistration.attestationData())); - confirmation.set(InstanceValidator.SAN_IPS_ATTRNAME, Certificates.getSubjectAlternativeNames(instanceRegistration.csr(), SubjectAlternativeName.Type.IP)); - confirmation.set(InstanceValidator.SAN_DNS_ATTRNAME, Certificates.getSubjectAlternativeNames(instanceRegistration.csr(), SubjectAlternativeName.Type.DNS)); - if (!instanceValidator.isValidInstance(confirmation)) { - log.log(Level.INFO, "Invalid instance registration for " + instanceRegistration.toString()); - return ErrorResponse.forbidden("Unable to launch service: " +instanceRegistration.service()); - } - var certificate = certificates.create(instanceRegistration.csr(), caCertificate(), caPrivateKey()); - var instanceId = Certificates.instanceIdFrom(instanceRegistration.csr()); - var identity = new InstanceIdentity(instanceRegistration.provider(), instanceRegistration.service(), instanceId, - Optional.of(certificate)); - return new SlimeJsonResponse(InstanceSerializer.identityToSlime(identity)); - } - - private HttpResponse refreshInstance(HttpRequest request, String provider, String service, String instanceId) { - var instanceRefresh = deserializeRequest(request, InstanceSerializer::refreshFromSlime); - var instanceIdFromCsr = Certificates.instanceIdFrom(instanceRefresh.csr()); - - var athenzService = getRequestAthenzService(request); - - if (!instanceIdFromCsr.equals(instanceId)) { - throw new IllegalArgumentException("Mismatch between instance ID in URL path and instance ID in CSR " + - "[instanceId=" + instanceId + ",instanceIdFromCsr=" + instanceIdFromCsr + - "]"); - } - - // Verify that the csr instance id matches one of the certificates in the chain - refreshesSameInstanceId(instanceIdFromCsr, request); - - - // Validate that there is no privilege escalation (can only refresh same service) - refreshesSameService(instanceRefresh, athenzService); - - InstanceConfirmation instanceConfirmation = new InstanceConfirmation(provider, athenzService.getDomain().getName(), athenzService.getName(), null); - instanceConfirmation.set(InstanceValidator.SAN_IPS_ATTRNAME, Certificates.getSubjectAlternativeNames(instanceRefresh.csr(), SubjectAlternativeName.Type.IP)); - instanceConfirmation.set(InstanceValidator.SAN_DNS_ATTRNAME, Certificates.getSubjectAlternativeNames(instanceRefresh.csr(), SubjectAlternativeName.Type.DNS)); - if(!instanceValidator.isValidRefresh(instanceConfirmation)) { - return ErrorResponse.forbidden("Unable to refresh cert: " + instanceRefresh.csr().getSubject().toString()); - } - - var certificate = certificates.create(instanceRefresh.csr(), caCertificate(), caPrivateKey()); - var identity = new InstanceIdentity(provider, service, instanceIdFromCsr, Optional.of(certificate)); - return new SlimeJsonResponse(InstanceSerializer.identityToSlime(identity)); - } - - public void refreshesSameInstanceId(String csrInstanceId, HttpRequest request) { - String certificateInstanceId = getRequestCertificateChain(request).stream() - .map(Certificates::instanceIdFrom) - .filter(Optional::isPresent) - .map(Optional::get) - .findAny().orElseThrow(() -> new IllegalArgumentException("No client certificate with instance id in request.")); - - if(! Objects.equals(certificateInstanceId, csrInstanceId)) { - throw new IllegalArgumentException("Mismatch between instance ID in client certificate and instance ID in CSR " + - "[instanceId=" + certificateInstanceId + ",instanceIdFromCsr=" + csrInstanceId + - "]"); - } - } - - private void refreshesSameService(InstanceRefresh instanceRefresh, AthenzService athenzService) { - List<String> commonNames = X509CertificateUtils.getCommonNames(instanceRefresh.csr().getSubject()); - if(commonNames.size() != 1 && !Objects.equals(commonNames.get(0), athenzService.getFullName())) { - throw new IllegalArgumentException(String.format("Invalid request, trying to refresh service %s using service %s.", instanceRefresh.csr().getSubject().getName(), athenzService.getFullName())); - } - } - - /** Returns CA certificate from secret store */ - private X509Certificate caCertificate() { - return X509CertificateUtils.fromPem(secretStore.getSecret(caCertificateSecretName)); - } - - private List<X509Certificate> getRequestCertificateChain(HttpRequest request) { - return Optional.ofNullable(request.getJDiscRequest().context().get(RequestUtils.JDISC_REQUEST_X509CERT)) - .map(X509Certificate[].class::cast) - .map(Arrays::asList) - .orElse(Collections.emptyList()); - } - - private AthenzService getRequestAthenzService(HttpRequest request) { - return getRequestCertificateChain(request).stream() - .findFirst() - .flatMap(X509CertificateUtils::getSubjectCommonName) - .map(AthenzService::new) - .orElseThrow(() -> new RuntimeException("No certificate found")); - } - - /** Returns CA private key from secret store */ - private PrivateKey caPrivateKey() { - return KeyUtils.fromPemEncodedPrivateKey(secretStore.getSecret(caPrivateKeySecretName)); - } - - private static <T> T deserializeRequest(HttpRequest request, Function<Slime, T> serializer) { - try { - var slime = SlimeUtils.jsonToSlime(request.getData().readAllBytes()); - return serializer.apply(slime); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java deleted file mode 100644 index fec03afab69..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.restapi; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.yahoo.security.Pkcs10CsrUtils; -import com.yahoo.security.X509CertificateUtils; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.text.StringUtilities; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.identityprovider.api.ClusterType; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; -import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import com.yahoo.vespa.hosted.ca.instance.InstanceIdentity; -import com.yahoo.vespa.hosted.ca.instance.InstanceRefresh; -import com.yahoo.vespa.hosted.ca.instance.InstanceRegistration; - -import java.io.IOException; -import java.time.Instant; -import java.util.HashSet; -import java.util.Set; - -/** - * @author mpolden - */ -public class InstanceSerializer { - - private static final String PROVIDER_FIELD = "provider"; - private static final String DOMAIN_FIELD = "domain"; - private static final String SERVICE_FIELD = "service"; - private static final String ATTESTATION_DATA_FIELD = "attestationData"; - private static final String CSR_FIELD = "csr"; - private static final String NAME_FIELD = "service"; - private static final String INSTANCE_ID_FIELD = "instanceId"; - private static final String X509_CERTIFICATE_FIELD = "x509Certificate"; - - private static final String IDD_SIGNATURE_FIELD = "signature"; - private static final String IDD_SIGNING_KEY_VERSION_FIELD = "signing-key-version"; - private static final String IDD_PROVIDER_UNIQUE_ID_FIELD = "provider-unique-id"; - private static final String IDD_PROVIDER_SERVICE_FIELD = "provider-service"; - private static final String IDD_DOCUMENT_VERSION_FIELD = "document-version"; - private static final String IDD_CONFIGSERVER_HOSTNAME_FIELD = "configserver-hostname"; - private static final String IDD_INSTANCE_HOSTNAME_FIELD = "instance-hostname"; - private static final String IDD_CREATED_AT_FIELD = "created-at"; - private static final String IDD_IPADDRESSES_FIELD = "ip-addresses"; - private static final String IDD_IDENTITY_TYPE_FIELD = "identity-type"; - private static final String IDD_CLUSTER_TYPE_FIELD = "cluster-type"; - - private static final ObjectMapper objectMapper = new ObjectMapper(); - static { - objectMapper.registerModule(new JavaTimeModule()); - } - - private InstanceSerializer() {} - - public static InstanceRegistration registrationFromSlime(Slime slime) { - Cursor root = slime.get(); - return new InstanceRegistration(requireField(PROVIDER_FIELD, root).asString(), - requireField(DOMAIN_FIELD, root).asString(), - requireField(SERVICE_FIELD, root).asString(), - attestationDataToIdentityDocument(StringUtilities.unescape(requireField(ATTESTATION_DATA_FIELD, root).asString())), - Pkcs10CsrUtils.fromPem(requireField(CSR_FIELD, root).asString())); - } - - public static InstanceRefresh refreshFromSlime(Slime slime) { - Cursor root = slime.get(); - return new InstanceRefresh(Pkcs10CsrUtils.fromPem(requireField(CSR_FIELD, root).asString())); - } - - public static Slime identityToSlime(InstanceIdentity identity) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString(PROVIDER_FIELD, identity.provider()); - root.setString(NAME_FIELD, identity.service()); - root.setString(INSTANCE_ID_FIELD, identity.instanceId()); - identity.x509Certificate() - .map(X509CertificateUtils::toPem) - .ifPresent(pem -> root.setString(X509_CERTIFICATE_FIELD, pem)); - return slime; - } - - public static SignedIdentityDocument attestationDataToIdentityDocument(String attestationData) { - Slime slime = SlimeUtils.jsonToSlime(attestationData); - Cursor root = slime.get(); - String signature = requireField(IDD_SIGNATURE_FIELD, root).asString(); - long signingKeyVersion = requireField(IDD_SIGNING_KEY_VERSION_FIELD, root).asLong(); - VespaUniqueInstanceId providerUniqueId = VespaUniqueInstanceId.fromDottedString(requireField(IDD_PROVIDER_UNIQUE_ID_FIELD, root).asString()); - AthenzService athenzService = new AthenzService(requireField(IDD_PROVIDER_SERVICE_FIELD, root).asString()); - long documentVersion = requireField(IDD_DOCUMENT_VERSION_FIELD, root).asLong(); - String configserverHostname = requireField(IDD_CONFIGSERVER_HOSTNAME_FIELD, root).asString(); - String instanceHostname = requireField(IDD_INSTANCE_HOSTNAME_FIELD, root).asString(); - double createdAtTimestamp = requireField(IDD_CREATED_AT_FIELD, root).asDouble(); - Instant createdAt = getJsr310Instant(createdAtTimestamp); - Set<String> ips = new HashSet<>(); - requireField(IDD_IPADDRESSES_FIELD, root).traverse((ArrayTraverser) (__, entry) -> ips.add(entry.asString())); - IdentityType identityType = IdentityType.fromId(requireField(IDD_IDENTITY_TYPE_FIELD, root).asString()); - var clusterTypeField = root.field(IDD_CLUSTER_TYPE_FIELD); - var clusterType = clusterTypeField.valid() ? ClusterType.from(clusterTypeField.asString()) : null; - - - return new SignedIdentityDocument(signature, (int)signingKeyVersion, providerUniqueId, athenzService, (int)documentVersion, - configserverHostname, instanceHostname, createdAt, ips, identityType, clusterType); - } - - private static Instant getJsr310Instant(double v) { - try { - return objectMapper.readValue(Double.toString(v), Instant.class); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private static Cursor requireField(String fieldName, Cursor root) { - var field = root.field(fieldName); - if (!field.valid()) throw new IllegalArgumentException("Missing required field '" + fieldName + "'"); - return field; - } - -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/package-info.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/package-info.java deleted file mode 100644 index 118f4b08c2a..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author mpolden - */ -@ExportPackage -package com.yahoo.vespa.hosted.ca.restapi; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AutoGeneratedKeyProvider.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AutoGeneratedKeyProvider.java deleted file mode 100644 index 67e5caa0c18..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AutoGeneratedKeyProvider.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyUtils; - -import java.security.KeyPair; -import java.security.KeyPairGenerator; -import java.security.NoSuchAlgorithmException; -import java.security.PrivateKey; -import java.security.PublicKey; - -/** - * @author bjorncs - */ -public class AutoGeneratedKeyProvider implements KeyProvider { - - private final KeyPair keyPair; - - public AutoGeneratedKeyProvider() { - keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); - } - - @Override - public PrivateKey getPrivateKey(int version) { - return keyPair.getPrivate(); - } - - @Override - public PublicKey getPublicKey(int version) { - return keyPair.getPublic(); - } - - public KeyPair getKeyPair() { - return keyPair; - } -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGeneratorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGeneratorTest.java deleted file mode 100644 index 9205baff0fc..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGeneratorTest.java +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.ClusterMembership; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; -import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import com.yahoo.vespa.athenz.identityprovider.client.IdentityDocumentSigner; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.Allocation; -import com.yahoo.vespa.hosted.provision.node.Generation; -import com.yahoo.vespa.hosted.provision.node.IP; -import com.yahoo.vespa.hosted.provision.node.Nodes; -import com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors; -import org.junit.jupiter.api.Test; - -import java.util.Optional; -import java.util.Set; - -import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.TestUtils.getAthenzProviderConfig; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author valerijf - */ -public class IdentityDocumentGeneratorTest { - - private static final Zone ZONE = new Zone(SystemName.cd, Environment.dev, RegionName.from("us-north-1")); - - @Test - void generates_valid_identity_document() { - String parentHostname = "docker-host"; - String containerHostname = "docker-container"; - - ApplicationId appid = ApplicationId.from( - TenantName.from("tenant"), ApplicationName.from("application"), InstanceName.from("default")); - Allocation allocation = new Allocation(appid, - ClusterMembership.from("container/default/0/0", Version.fromString("1.2.3"), Optional.empty()), - new NodeResources(1, 1, 1, 1), - Generation.initial(), - false); - Node parentNode = Node.create("ostkid", - IP.Config.ofEmptyPool(Set.of("127.0.0.1")), - parentHostname, - new MockNodeFlavors().getFlavorOrThrow("default"), - NodeType.host).build(); - Node containerNode = Node.reserve(Set.of("::1"), - containerHostname, - parentHostname, - new MockNodeFlavors().getFlavorOrThrow("default").resources(), - NodeType.tenant) - .allocation(allocation).build(); - NodeRepository nodeRepository = mock(NodeRepository.class); - Nodes nodes = mock(Nodes.class); - when(nodeRepository.nodes()).thenReturn(nodes); - - when(nodes.node(eq(parentHostname))).thenReturn(Optional.of(parentNode)); - when(nodes.node(eq(containerHostname))).thenReturn(Optional.of(containerNode)); - AutoGeneratedKeyProvider keyProvider = new AutoGeneratedKeyProvider(); - - String dnsSuffix = "vespa.dns.suffix"; - AthenzProviderServiceConfig config = getAthenzProviderConfig("domain", "service", dnsSuffix); - IdentityDocumentGenerator identityDocumentGenerator = - new IdentityDocumentGenerator(config, nodeRepository, ZONE, keyProvider); - SignedIdentityDocument signedIdentityDocument = identityDocumentGenerator.generateSignedIdentityDocument(containerHostname, IdentityType.TENANT); - - // Verify attributes - assertEquals(containerHostname, signedIdentityDocument.instanceHostname()); - - String environment = "dev"; - String region = "us-north-1"; - - VespaUniqueInstanceId expectedProviderUniqueId = - new VespaUniqueInstanceId(0, "default", "default", "application", "tenant", region, environment, IdentityType.TENANT); - assertEquals(expectedProviderUniqueId, signedIdentityDocument.providerUniqueId()); - - // Validate that container ips are present - assertTrue(signedIdentityDocument.ipAddresses().contains("::1")); - - IdentityDocumentSigner signer = new IdentityDocumentSigner(); - - // Validate signature - assertTrue(signer.hasValidSignature(signedIdentityDocument, keyProvider.getPublicKey(0))); - } -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidatorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidatorTest.java deleted file mode 100644 index a7947aff283..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidatorTest.java +++ /dev/null @@ -1,296 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.yahoo.component.Version; -import com.yahoo.config.model.api.ApplicationInfo; -import com.yahoo.config.model.api.HostInfo; -import com.yahoo.config.model.api.Model; -import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.config.model.api.SuperModel; -import com.yahoo.config.model.api.SuperModelProvider; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterMembership; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.identityprovider.api.ClusterType; -import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; -import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import com.yahoo.vespa.athenz.identityprovider.client.IdentityDocumentSigner; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceValidator.ValidationException; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeList; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.IP; -import com.yahoo.vespa.hosted.provision.node.Nodes; -import com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors; -import org.junit.jupiter.api.Test; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; - -import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceValidator.SERVICE_PROPERTIES_DOMAIN_KEY; -import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceValidator.SERVICE_PROPERTIES_SERVICE_KEY; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author valerijf - * @author bjorncs - * @author mortent - */ -public class InstanceValidatorTest { - - private final ApplicationId applicationId = ApplicationId.from("tenant", "application", "instance"); - private final String domain = "domain"; - private final String service = "service"; - - private final AthenzService vespaTenantDomain = new AthenzService("vespa.vespa.tenant"); - private final AutoGeneratedKeyProvider keyProvider = new AutoGeneratedKeyProvider(); - - @Test - void application_does_not_exist() { - SuperModelProvider superModelProvider = mockSuperModelProvider(); - InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, null, vespaTenantDomain); - assertFalse(instanceValidator.isValidInstance(createRegisterInstanceConfirmation(applicationId, domain, service))); - } - - @Test - void application_does_not_have_domain_set() { - SuperModelProvider superModelProvider = mockSuperModelProvider( - mockApplicationInfo(applicationId, 5, Collections.emptyList())); - InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, new IdentityDocumentSigner(), vespaTenantDomain); - - assertFalse(instanceValidator.isValidInstance(createRegisterInstanceConfirmation(applicationId, domain, service))); - } - - @Test - void application_has_wrong_domain() { - ServiceInfo serviceInfo = new ServiceInfo("serviceName", "type", Collections.emptyList(), - Collections.singletonMap(SERVICE_PROPERTIES_DOMAIN_KEY, "not-domain"), "confId", "hostName"); - - SuperModelProvider superModelProvider = mockSuperModelProvider( - mockApplicationInfo(applicationId, 5, Collections.singletonList(serviceInfo))); - InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, null, vespaTenantDomain); - - assertFalse(instanceValidator.isValidInstance(createRegisterInstanceConfirmation(applicationId, domain, service))); - } - - @Test - void application_has_same_domain_and_service() { - Map<String, String> properties = new HashMap<>(); - properties.put(SERVICE_PROPERTIES_DOMAIN_KEY, domain); - properties.put(SERVICE_PROPERTIES_SERVICE_KEY, service); - - ServiceInfo serviceInfo = new ServiceInfo("serviceName", "type", Collections.emptyList(), - properties, "confId", "hostName"); - - SuperModelProvider superModelProvider = mockSuperModelProvider( - mockApplicationInfo(applicationId, 5, Collections.singletonList(serviceInfo))); - IdentityDocumentSigner signer = mock(IdentityDocumentSigner.class); - when(signer.hasValidSignature(any(), any())).thenReturn(true); - InstanceValidator instanceValidator = new InstanceValidator(mock(KeyProvider.class), superModelProvider, mockNodeRepo(), signer, vespaTenantDomain); - - assertTrue(instanceValidator.isValidInstance(createRegisterInstanceConfirmation(applicationId, domain, service))); - } - - @Test - void rejects_invalid_provider_unique_id_in_csr() { - SuperModelProvider superModelProvider = mockSuperModelProvider(); - InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, null, vespaTenantDomain); - InstanceConfirmation instanceConfirmation = createRegisterInstanceConfirmation(applicationId, domain, service); - VespaUniqueInstanceId tamperedId = new VespaUniqueInstanceId(0, "default", "instance", "app", "tenant", "us-north-1", "dev", IdentityType.NODE); - instanceConfirmation.set("sanDNS", tamperedId.asDottedString() + ".instanceid.athenz.dev-us-north-1.vespa.yahoo.cloud"); - assertFalse(instanceValidator.isValidInstance(instanceConfirmation)); - } - - @Test - void rejects_unknown_ips_in_csr() { - NodeRepository nodeRepository = mockNodeRepo(); - InstanceValidator instanceValidator = new InstanceValidator(null, mockSuperModelProvider(), nodeRepository, null, vespaTenantDomain); - InstanceConfirmation instanceConfirmation = createRegisterInstanceConfirmation(applicationId, domain, service); - Set<String> nodeIp = nodeRepository.nodes().list().owner(applicationId).stream().findFirst() - .map(Node::ipConfig) - .map(IP.Config::primary) - .orElseThrow(() -> new RuntimeException("No ipaddress for mocked node")); - - List<String> ips = new ArrayList<>(nodeIp); - ips.add("::ff"); - instanceConfirmation.set("sanIP", String.join(",", ips)); - assertFalse(instanceValidator.isValidInstance(instanceConfirmation)); - } - - @Test - void rejects_invalid_cluster_type_in_csr() { - var props = Map.of(SERVICE_PROPERTIES_DOMAIN_KEY, domain, SERVICE_PROPERTIES_SERVICE_KEY, service); - var info = new ServiceInfo("serviceName", "type", List.of(), props, "confId", "hostName"); - var provider = mockSuperModelProvider(mockApplicationInfo(applicationId, 5, List.of(info))); - var instanceValidator = new InstanceValidator(keyProvider, provider, mockNodeRepo(), new IdentityDocumentSigner(), vespaTenantDomain); - var instanceConfirmation = createRegisterInstanceConfirmation(applicationId, domain, service); - instanceConfirmation.set("sanURI", "vespa://cluster-type/content"); - var exception = assertThrows(ValidationException.class, () -> instanceValidator.validateInstance(instanceConfirmation)); - var expectedMsg = "Illegal SAN URIs: expected '[vespa://cluster-type/container]' found '[vespa://cluster-type/content]'"; - assertEquals(expectedMsg, exception.getMessage()); - } - - @Test - void accepts_valid_refresh_requests() { - NodeRepository nodeRepository = mock(NodeRepository.class); - Nodes nodes = mock(Nodes.class); - when(nodeRepository.nodes()).thenReturn(nodes); - InstanceValidator instanceValidator = new InstanceValidator(null, null, nodeRepository, new IdentityDocumentSigner(), vespaTenantDomain); - - List<Node> nodeList = createNodes(10); - Node node = nodeList.get(0); - nodeList = allocateNode(nodeList, node, applicationId); - when(nodes.list()).thenReturn(NodeList.copyOf(nodeList)); - String nodeIp = node.ipConfig().primary().stream().findAny().orElseThrow(() -> new RuntimeException("No ipaddress for mocked node")); - InstanceConfirmation instanceConfirmation = createRefreshInstanceConfirmation(applicationId, domain, service, List.of(nodeIp)); - - assertTrue(instanceValidator.isValidRefresh(instanceConfirmation)); - } - - @Test - void rejects_refresh_on_ip_mismatch() { - NodeRepository nodeRepository = mockNodeRepo(); - InstanceValidator instanceValidator = new InstanceValidator(null, null, nodeRepository, new IdentityDocumentSigner(), vespaTenantDomain); - - Set<String> nodeIp = nodeRepository.nodes().list().owner(applicationId).stream().findFirst() - .map(Node::ipConfig) - .map(IP.Config::primary) - .orElseThrow(() -> new RuntimeException("No ipaddress for mocked node")); - - List<String> ips = new ArrayList<>(nodeIp); - ips.add("::ff"); - // Add invalid ip to list of ip addresses - InstanceConfirmation instanceConfirmation = createRefreshInstanceConfirmation(applicationId, domain, service, ips); - - assertFalse(instanceValidator.isValidRefresh(instanceConfirmation)); - } - - @Test - void rejects_refresh_when_node_is_not_allocated() { - NodeRepository nodeRepository = mock(NodeRepository.class); - Nodes nodes = mock(Nodes.class); - when(nodeRepository.nodes()).thenReturn(nodes); - - InstanceValidator instanceValidator = new InstanceValidator(null, null, nodeRepository, new IdentityDocumentSigner(), vespaTenantDomain); - - List<Node> nodeList = createNodes(10); - - when(nodes.list()).thenReturn(NodeList.copyOf(nodeList)); - InstanceConfirmation instanceConfirmation = createRefreshInstanceConfirmation(applicationId, domain, service, List.of("::11")); - - assertFalse(instanceValidator.isValidRefresh(instanceConfirmation)); - - } - - private NodeRepository mockNodeRepo() { - NodeRepository nodeRepository = mock(NodeRepository.class); - Nodes nodes = mock(Nodes.class); - when(nodeRepository.nodes()).thenReturn(nodes); - List<Node> nodeList = createNodes(10); - Node node = nodeList.get(0); - nodeList = allocateNode(nodeList, node, applicationId); - when(nodes.list()).thenReturn(NodeList.copyOf(nodeList)); - return nodeRepository; - } - - private InstanceConfirmation createRegisterInstanceConfirmation( - ApplicationId applicationId, String domain, String service) { - VespaUniqueInstanceId vespaUniqueInstanceId = new VespaUniqueInstanceId(0, "default", applicationId.instance().value(), applicationId.application().value(), applicationId.tenant().value(), "us-north-1", "dev", IdentityType.NODE); - var domainService = new AthenzService(domain, service); - var clock = Instant.now(); - var clusterType = ClusterType.CONTAINER; - var signature = new IdentityDocumentSigner() - .generateSignature( - vespaUniqueInstanceId, domainService, "localhost", "localhost", clock, Set.of(), - IdentityType.NODE, keyProvider.getPrivateKey(0)); - SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument( - signature, 0, vespaUniqueInstanceId, domainService, 0, "localhost", "localhost", - clock, Collections.emptySet(), IdentityType.NODE, clusterType); - return createInstanceConfirmation(vespaUniqueInstanceId, domain, service, signedIdentityDocument); - } - - private InstanceConfirmation createRefreshInstanceConfirmation(ApplicationId applicationId, String domain, String service, List<String> ips) { - VespaUniqueInstanceId vespaUniqueInstanceId = new VespaUniqueInstanceId(0, "default", applicationId.instance().value(), applicationId.application().value(), applicationId.tenant().value(), "us-north-1", "dev", IdentityType.NODE); - InstanceConfirmation instanceConfirmation = createInstanceConfirmation(vespaUniqueInstanceId, domain, service, null); - instanceConfirmation.set("sanIP", String.join(",", ips)); - return instanceConfirmation; - } - - private InstanceConfirmation createInstanceConfirmation(VespaUniqueInstanceId vespaUniqueInstanceId, String domain, String service, SignedIdentityDocument identityDocument) { - InstanceConfirmation instanceConfirmation = new InstanceConfirmation( - "vespa.vespa.cd.provider_dev_us-north-1", - domain, - service, - Optional.ofNullable(identityDocument) - .map(EntityBindingsMapper::toSignedIdentityDocumentEntity) - .orElse(null)); - instanceConfirmation.set("sanDNS", vespaUniqueInstanceId.asDottedString() + ".instanceid.athenz.dev-us-north-1.vespa.yahoo.cloud"); - instanceConfirmation.set("sanURI", "vespa://cluster-type/container"); - return instanceConfirmation; - } - - private SuperModelProvider mockSuperModelProvider(ApplicationInfo... appInfos) { - SuperModel superModel = new SuperModel(Stream.of(appInfos) - .collect(Collectors.toMap( - ApplicationInfo::getApplicationId, - Function.identity() - ) - ), - true); - - SuperModelProvider superModelProvider = mock(SuperModelProvider.class); - when(superModelProvider.getSuperModel()).thenReturn(superModel); - return superModelProvider; - } - - private ApplicationInfo mockApplicationInfo(ApplicationId appId, int numHosts, List<ServiceInfo> serviceInfo) { - List<HostInfo> hosts = IntStream.range(0, numHosts) - .mapToObj(i -> new HostInfo("host-" + i + "." + appId.toShortString() + ".yahoo.com", serviceInfo)) - .toList(); - - Model model = mock(Model.class); - when(model.getHosts()).thenReturn(hosts); - - return new ApplicationInfo(appId, 0, model); - } - - private List<Node> createNodes(int num) { - MockNodeFlavors flavors = new MockNodeFlavors(); - List<Node> nodeList = new ArrayList<>(); - for (int i = 0; i < num; i++) { - Node node = Node.create("foo" + i, new IP.Config(Set.of("::1" + i, "::2" + i, "::3" + i), Set.of()), - "foo" + i, flavors.getFlavorOrThrow("default"), NodeType.tenant).build(); - nodeList.add(node); - } - return nodeList; - } - - private List<Node> allocateNode(List<Node> nodeList, Node node, ApplicationId applicationId) { - nodeList.removeIf(n -> n.id().equals(node.id())); - nodeList.add(node.allocate(applicationId, - ClusterMembership.from("container/default/0/0", Version.fromString("6.123.4"), Optional.empty()), - new NodeResources(1, 1, 1, 1), - Instant.now())); - return nodeList; - } -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/TestUtils.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/TestUtils.java deleted file mode 100644 index 4110ad2bfa2..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/TestUtils.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; - -/** - * @author bjorncs - */ -public class TestUtils { - - public static AthenzProviderServiceConfig getAthenzProviderConfig(String domain, - String service, - String dnsSuffix) { - AthenzProviderServiceConfig.Builder zoneConfig = - new AthenzProviderServiceConfig.Builder() - .serviceName(service) - .secretVersion(0) - .domain(domain) - .certDnsSuffix(dnsSuffix) - .ztsUrl("localhost/zts") - .secretName("s3cr3t") - .caCertSecretName(domain + ".ca.cert"); - return new AthenzProviderServiceConfig( - zoneConfig.athenzCaTrustStore("/dummy/path/to/athenz-ca.jks")); - } - -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificateTester.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificateTester.java deleted file mode 100644 index 4012776949e..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificateTester.java +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca; - -import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.Pkcs10Csr; -import com.yahoo.security.Pkcs10CsrBuilder; -import com.yahoo.security.SignatureAlgorithm; -import com.yahoo.security.SubjectAlternativeName; -import com.yahoo.security.X509CertificateBuilder; - -import javax.security.auth.x500.X500Principal; -import java.math.BigInteger; -import java.security.KeyPair; -import java.security.cert.X509Certificate; -import java.time.Duration; -import java.time.Instant; -import java.util.List; - -import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; - -/** - * Helper class for creating certificates, CSRs etc. for testing purposes. - * - * @author mpolden - */ -public class CertificateTester { - - private CertificateTester() {} - - public static X509Certificate createCertificate() { - var keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); - return createCertificate("subject", keyPair); - } - - public static X509Certificate createCertificate(String cn, KeyPair keyPair) { - var subject = new X500Principal("CN=" + cn); - return X509CertificateBuilder.fromKeypair(keyPair, - subject, - Instant.EPOCH, - Instant.EPOCH.plus(Duration.ofMinutes(1)), - SHA256_WITH_ECDSA, - BigInteger.ONE) - .build(); - } - - public static Pkcs10Csr createCsr() { - return createCsr(List.of(), List.of()); - } - - public static Pkcs10Csr createCsr(String dnsName) { - return createCsr(List.of(dnsName), List.of()); - } - - public static Pkcs10Csr createCsr(List<String> dnsNames) { - return createCsr(dnsNames, List.of()); - } - - public static Pkcs10Csr createCsr(String cn, List<String> dnsNames) { - return createCsr(cn, dnsNames, List.of()); - } - - public static Pkcs10Csr createCsr(List<String> dnsNames, List<String> ipAddresses) { - return createCsr("subject", dnsNames, ipAddresses); - } - public static Pkcs10Csr createCsr(String cn, List<String> dnsNames, List<String> ipAddresses) { - X500Principal subject = new X500Principal("CN=" + cn); - KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); - var builder = Pkcs10CsrBuilder.fromKeypair(subject, keyPair, SignatureAlgorithm.SHA512_WITH_ECDSA); - for (var dnsName : dnsNames) { - builder = builder.addSubjectAlternativeName(SubjectAlternativeName.Type.DNS, dnsName); - } - for (var ipAddress : ipAddresses) { - builder = builder.addSubjectAlternativeName(SubjectAlternativeName.Type.IP, ipAddress); - } - return builder.build(); - } - -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificatesTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificatesTest.java deleted file mode 100644 index dd3ddeeb804..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificatesTest.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca; - -import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.SubjectAlternativeName; -import com.yahoo.test.ManualClock; -import org.junit.jupiter.api.Test; - -import java.security.KeyPair; -import java.security.cert.X509Certificate; -import java.time.Duration; -import java.util.List; - -import static java.time.temporal.ChronoUnit.SECONDS; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - -/** - * @author mpolden - */ -public class CertificatesTest { - - private final KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); - private final X509Certificate caCertificate = CertificateTester.createCertificate("CA", keyPair); - - @Test - void expiry() { - var clock = new ManualClock(); - var certificates = new Certificates(clock); - var csr = CertificateTester.createCsr(); - var certificate = certificates.create(csr, caCertificate, keyPair.getPrivate()); - var now = clock.instant(); - - assertEquals(now.minus(Duration.ofHours(1)).truncatedTo(SECONDS), certificate.getNotBefore().toInstant()); - assertEquals(now.plus(Duration.ofDays(30)).truncatedTo(SECONDS), certificate.getNotAfter().toInstant()); - } - - @Test - void add_san_from_csr() throws Exception { - var certificates = new Certificates(new ManualClock()); - var dnsName = "host.example.com"; - var ip = "192.0.2.42"; - var csr = CertificateTester.createCsr(List.of(dnsName), List.of(ip)); - var certificate = certificates.create(csr, caCertificate, keyPair.getPrivate()); - - assertNotNull(certificate.getSubjectAlternativeNames()); - assertEquals(2, certificate.getSubjectAlternativeNames().size()); - - var subjectAlternativeNames = List.copyOf(certificate.getSubjectAlternativeNames()); - assertEquals(List.of(SubjectAlternativeName.Type.DNS.getTag(), dnsName), - subjectAlternativeNames.get(0)); - assertEquals(List.of(SubjectAlternativeName.Type.IP.getTag(), ip), - subjectAlternativeNames.get(1)); - } - - @Test - void parse_instance_id() { - var instanceId = "1.cluster1.default.app1.tenant1.us-north-1.prod.node"; - var instanceIdWithSuffix = instanceId + ".instanceid.athenz.dev-us-north-1.vespa.aws.oath.cloud"; - var csr = CertificateTester.createCsr(List.of("foo", "bar", instanceIdWithSuffix)); - assertEquals(instanceId, Certificates.instanceIdFrom(csr)); - } - -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java deleted file mode 100644 index bf2115e8759..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.restapi; - -import com.yahoo.application.container.handler.Request; -import com.yahoo.jdisc.http.server.jetty.RequestUtils; -import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.Pkcs10Csr; -import com.yahoo.security.Pkcs10CsrUtils; -import com.yahoo.security.X509CertificateUtils; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.text.StringUtilities; -import com.yahoo.vespa.athenz.api.AthenzPrincipal; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.client.ErrorHandler; -import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; -import com.yahoo.vespa.hosted.ca.CertificateTester; -import org.apache.http.client.ResponseHandler; -import org.apache.http.client.methods.HttpUriRequest; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import javax.net.ssl.SSLContext; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.security.Principal; -import java.security.cert.X509Certificate; -import java.util.List; -import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - -/** - * @author mpolden - */ -public class CertificateAuthorityApiTest extends ContainerTester { - - private static final String INSTANCE_ID = "1.cluster1.default.app1.tenant1.us-north-1.prod.node"; - private static final String INSTANCE_ID_WITH_SUFFIX = INSTANCE_ID + ".instanceid.athenz.dev-us-north-1.vespa.aws.oath.cloud"; - private static final String INVALID_INSTANCE_ID = "1.cluster1.default.otherapp.othertenant.us-north-1.prod.node"; - private static final String INVALID_INSTANCE_ID_WITH_SUFFIX = INVALID_INSTANCE_ID + ".instanceid.athenz.dev-us-north-1.vespa.aws.oath.cloud"; - - private static final String CONTAINER_IDENTITY = "vespa.external.tenant"; - private static final String HOST_IDENTITY = "vespa.external.tenant-host"; - - @BeforeEach - public void before() { - setCaCertificateAndKey(); - } - - @Test - void register_instance() throws Exception { - // POST instance registration - var csr = CertificateTester.createCsr(List.of("node1.example.com", INSTANCE_ID_WITH_SUFFIX)); - assertIdentityResponse(new Request("http://localhost:12345/ca/v1/instance/", - instanceRegistrationJson(csr), - Request.Method.POST)); - - // POST instance registration with ZTS client - var ztsClient = new TestZtsClient(new AthenzPrincipal(new AthenzService(HOST_IDENTITY)), null, URI.create("http://localhost:12345/ca/v1/"), SSLContext.getDefault()); - var instanceIdentity = ztsClient.registerInstance(new AthenzService("vespa.external", "provider_prod_us-north-1"), - new AthenzService(CONTAINER_IDENTITY), - getAttestationData(), - csr); - assertEquals("CN=Vespa CA", instanceIdentity.certificate().getIssuerX500Principal().getName()); - } - - private X509Certificate registerInstance() throws Exception { - // POST instance registration - var csr = CertificateTester.createCsr(CONTAINER_IDENTITY, List.of("node1.example.com", INSTANCE_ID_WITH_SUFFIX)); - assertIdentityResponse(new Request("http://localhost:12345/ca/v1/instance/", - instanceRegistrationJson(csr), - Request.Method.POST)); - - // POST instance registration with ZTS client - var ztsClient = new TestZtsClient(new AthenzPrincipal(new AthenzService(HOST_IDENTITY)), null, URI.create("http://localhost:12345/ca/v1/"), SSLContext.getDefault()); - var instanceIdentity = ztsClient.registerInstance(new AthenzService("vespa.external", "provider_prod_us-north-1"), - new AthenzService(CONTAINER_IDENTITY), - getAttestationData(), - csr); - return instanceIdentity.certificate(); - } - - @Test - void refresh_instance() throws Exception { - // Register instance to get cert - var certificate = registerInstance(); - - // POST instance refresh - var principal = new AthenzPrincipal(new AthenzService(CONTAINER_IDENTITY)); - var csr = CertificateTester.createCsr(principal.getIdentity().getFullName(), List.of("node1.example.com", INSTANCE_ID_WITH_SUFFIX)); - var request = new Request("http://localhost:12345/ca/v1/instance/vespa.external.provider_prod_us-north-1/vespa.external/tenant/" + INSTANCE_ID, - instanceRefreshJson(csr), - Request.Method.POST, - principal); - request.getAttributes().put(RequestUtils.JDISC_REQUEST_X509CERT, new X509Certificate[]{certificate}); - assertIdentityResponse(request); - - // POST instance refresh with ZTS client - var ztsClient = new TestZtsClient(principal, certificate, URI.create("http://localhost:12345/ca/v1/"), SSLContext.getDefault()); - var instanceIdentity = ztsClient.refreshInstance(new AthenzService("vespa.external", "provider_prod_us-north-1"), - new AthenzService(CONTAINER_IDENTITY), - INSTANCE_ID, - csr); - assertEquals("CN=Vespa CA", instanceIdentity.certificate().getIssuerX500Principal().getName()); - } - - @Test - void invalid_requests() throws Exception { - // POST instance registration with missing fields - assertResponse(400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"POST http://localhost:12345/ca/v1/instance/ failed: Missing required field 'provider'\"}", - new Request("http://localhost:12345/ca/v1/instance/", - new byte[0], - Request.Method.POST)); - - // POST instance registration without DNS name in CSR - var csr = CertificateTester.createCsr(); - var request = new Request("http://localhost:12345/ca/v1/instance/", - instanceRegistrationJson(csr), - Request.Method.POST); - assertResponse(400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"POST http://localhost:12345/ca/v1/instance/ failed: No instance ID found in CSR\"}", request); - - // POST instance refresh with missing field - assertResponse(400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"POST http://localhost:12345/ca/v1/instance/vespa.external.provider_prod_us-north-1/vespa.external/tenant/1.cluster1.default.app1.tenant1.us-north-1.prod.node failed: Missing required field 'csr'\"}", - new Request("http://localhost:12345/ca/v1/instance/vespa.external.provider_prod_us-north-1/vespa.external/tenant/" + INSTANCE_ID, - new byte[0], - Request.Method.POST)); - - // POST instance refresh where instanceId does not match CSR dnsName - var principal = new AthenzPrincipal(new AthenzService(CONTAINER_IDENTITY)); - var cert = CertificateTester.createCertificate(CONTAINER_IDENTITY, KeyUtils.generateKeypair(KeyAlgorithm.EC)); - csr = CertificateTester.createCsr(principal.getIdentity().getFullName(), List.of("node1.example.com", INSTANCE_ID_WITH_SUFFIX)); - request = new Request("http://localhost:12345/ca/v1/instance/vespa.external.provider_prod_us-north-1/vespa.external/tenant/foobar", - instanceRefreshJson(csr), - Request.Method.POST, - principal); - request.getAttributes().put(RequestUtils.JDISC_REQUEST_X509CERT, new X509Certificate[]{cert}); - assertResponse( - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"POST http://localhost:12345/ca/v1/instance/vespa.external.provider_prod_us-north-1/vespa.external/tenant/foobar failed: Mismatch between instance ID in URL path and instance ID in CSR [instanceId=foobar,instanceIdFromCsr=1.cluster1.default.app1.tenant1.us-north-1.prod.node]\"}", - request); - - // POST instance refresh using zts client where client cert does not contain instanceid - var certificate = registerInstance(); - var ztsClient = new TestZtsClient(principal, certificate, URI.create("http://localhost:12345/ca/v1/"), SSLContext.getDefault()); - try { - var invalidCsr = CertificateTester.createCsr(principal.getIdentity().getFullName(), List.of("node1.example.com", INVALID_INSTANCE_ID_WITH_SUFFIX)); - var instanceIdentity = ztsClient.refreshInstance(new AthenzService("vespa.external", "provider_prod_us-north-1"), - new AthenzService(CONTAINER_IDENTITY), - INSTANCE_ID, - invalidCsr); - fail("Refresh instance should have failed"); - } catch (Exception e) { - String expectedMessage = "Received error from ZTS: code=0, message=\"POST http://localhost:12345/ca/v1/instance/vespa.external.provider_prod_us-north-1/vespa.external/tenant/1.cluster1.default.app1.tenant1.us-north-1.prod.node failed: Mismatch between instance ID in URL path and instance ID in CSR [instanceId=1.cluster1.default.app1.tenant1.us-north-1.prod.node,instanceIdFromCsr=1.cluster1.default.otherapp.othertenant.us-north-1.prod.node]\""; - assertEquals(expectedMessage, e.getMessage()); - } - } - - private void setCaCertificateAndKey() { - var keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); - var caCertificatePem = X509CertificateUtils.toPem(CertificateTester.createCertificate("Vespa CA", keyPair)); - var privateKeyPem = KeyUtils.toPem(keyPair.getPrivate()); - secretStore().setSecret("vespa.external.ca.cert", caCertificatePem) - .setSecret("secretname", privateKeyPem); - } - - private void assertIdentityResponse(Request request) { - assertResponse(200, (body) -> { - var slime = SlimeUtils.jsonToSlime(body); - var root = slime.get(); - assertEquals("vespa.external.provider_prod_us-north-1", root.field("provider").asString()); - assertEquals("tenant", root.field("service").asString()); - assertEquals(INSTANCE_ID, root.field("instanceId").asString()); - var pemEncodedCertificate = root.field("x509Certificate").asString(); - assertTrue(pemEncodedCertificate.startsWith("-----BEGIN CERTIFICATE-----") && - pemEncodedCertificate.endsWith("-----END CERTIFICATE-----\n"), - "Response contains PEM certificate"); - }, request); - } - - private static byte[] instanceRefreshJson(Pkcs10Csr csr) { - var csrPem = Pkcs10CsrUtils.toPem(csr); - var json = "{\"csr\": \"" + csrPem + "\"}"; - return json.getBytes(StandardCharsets.UTF_8); - } - - private static byte[] instanceRegistrationJson(Pkcs10Csr csr) { - var csrPem = Pkcs10CsrUtils.toPem(csr); - var json = "{\n" + - " \"provider\": \"vespa.external.provider_prod_us-north-1\",\n" + - " \"domain\": \"vespa.external\",\n" + - " \"service\": \"tenant\",\n" + - " \"attestationData\": \""+getAttestationData()+"\",\n" + - " \"csr\": \"" + csrPem + "\"\n" + - "}"; - return json.getBytes(StandardCharsets.UTF_8); - } - - private static String getAttestationData () { - var json = "{\n" + - " \"signature\": \"SIGNATURE\",\n" + - " \"signing-key-version\": 0,\n" + - " \"provider-unique-id\": \"0.default.default.application.tenant.us-north-1.dev.tenant\",\n" + - " \"provider-service\": \"domain.service\",\n" + - " \"document-version\": 1,\n" + - " \"configserver-hostname\": \"localhost\",\n" + - " \"instance-hostname\": \"docker-container\",\n" + - " \"created-at\": 1572000079.00000,\n" + - " \"ip-addresses\": [\n" + - " \"::1\"\n" + - " ],\n" + - " \"identity-type\": \"tenant\"\n" + - "}"; - return StringUtilities.escape(json); - } - - /* - Zts client that adds principal as header (since setting up ssl in test is cumbersome) - */ - private static class TestZtsClient extends DefaultZtsClient { - - private final Principal principal; - private final X509Certificate certificate; - - public TestZtsClient(Principal principal, X509Certificate certificate, URI ztsUrl, SSLContext sslContext) { - super(ztsUrl, () -> sslContext, null, ErrorHandler.empty()); - this.principal = principal; - this.certificate = certificate; - } - - @Override - protected <T> T execute(HttpUriRequest request, ResponseHandler<T> responseHandler) { - request.addHeader("PRINCIPAL", principal.getName()); - Optional.ofNullable(certificate).ifPresent(cert -> { - var pem = X509CertificateUtils.toPem(certificate); - request.addHeader("CERTIFICATE", StringUtilities.escape(pem)); - }); - return super.execute(request, responseHandler); - } - } -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java deleted file mode 100644 index 8112f5779e5..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.restapi; - -import com.yahoo.application.Networking; -import com.yahoo.application.container.JDisc; -import com.yahoo.application.container.handler.Request; -import com.yahoo.vespa.hosted.ca.restapi.mock.SecretStoreMock; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; - -import java.io.UncheckedIOException; -import java.nio.charset.CharacterCodingException; -import java.util.function.Consumer; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * The superclass of REST API tests which require a functional container instance. - * - * @author mpolden - */ -public class ContainerTester { - - private JDisc container; - - @BeforeEach - public void startContainer() { - container = JDisc.fromServicesXml(servicesXml(), Networking.enable); - } - - @AfterEach - public void stopContainer() { - container.close(); - } - - public SecretStoreMock secretStore() { - return (SecretStoreMock) container.components().getComponent(SecretStoreMock.class.getName()); - } - - public void assertResponse(int expectedStatus, String expectedBody, Request request) { - assertResponse(expectedStatus, (body) -> assertEquals(expectedBody, body), request); - } - - public void assertResponse(int expectedStatus, Consumer<String> bodyAsserter, Request request) { - var response = container.handleRequest(request); - try { - bodyAsserter.accept(response.getBodyAsString()); - } catch (CharacterCodingException e) { - throw new UncheckedIOException(e); - } - assertEquals(expectedStatus, response.getStatus()); - assertEquals("application/json; charset=UTF-8", response.getHeaders().getFirst("Content-Type")); - } - - private static String servicesXml() { - return "<container version='1.0'>\n" + - " <accesslog type=\"disabled\"/>\n" + - " <config name=\"container.handler.threadpool\">\n" + - " <maxthreads>10</maxthreads>\n" + - " </config>\n" + - " <config name='vespa.hosted.athenz.instanceproviderservice.config.athenz-provider-service'>\n" + - " <athenzCaTrustStore>/path/to/file</athenzCaTrustStore>\n" + - " <domain>vespa.external</domain>\n" + - " <serviceName>servicename</serviceName>\n" + - " <secretName>secretname</secretName>\n" + - " <secretVersion>0</secretVersion>\n" + - " <caCertSecretName>vespa.external.ca.cert</caCertSecretName>\n" + - " <certDnsSuffix>suffix</certDnsSuffix>\n" + - " <ztsUrl>https://localhost:123/</ztsUrl>\n" + - " </config>\n" + - " <component id='com.yahoo.vespa.hosted.ca.restapi.mock.SecretStoreMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.ca.restapi.mock.InstanceValidatorMock'/>\n" + - " <handler id='com.yahoo.vespa.hosted.ca.restapi.CertificateAuthorityApiHandler'>\n" + - " <binding>http://*/ca/v1/*</binding>\n" + - " </handler>\n" + - " <http>\n" + - " <server id='default' port='12345'/>\n" + - " <filtering>\n" + - " <request-chain id=\"my-default-chain\">\n" + - " <filter id='com.yahoo.vespa.hosted.ca.restapi.mock.PrincipalFromHeaderFilter' />\n" + - " <binding>http://*/*</binding>\n" + - " </request-chain>\n" + - " </filtering>\n" + - " </http>\n" + - "</container>"; - } - -}
\ No newline at end of file diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java deleted file mode 100644 index ca624918beb..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.restapi; - -import com.yahoo.security.Pkcs10CsrUtils; -import com.yahoo.security.X509CertificateUtils; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.text.StringUtilities; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.identityprovider.api.ClusterType; -import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; -import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import com.yahoo.vespa.hosted.ca.CertificateTester; -import com.yahoo.vespa.hosted.ca.instance.InstanceIdentity; -import com.yahoo.vespa.hosted.ca.instance.InstanceRefresh; -import com.yahoo.vespa.hosted.ca.instance.InstanceRegistration; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.Collections; -import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author mpolden - */ -public class InstanceSerializerTest { - - @Test - void deserialize_instance_registration() { - var csr = CertificateTester.createCsr(); - var csrPem = Pkcs10CsrUtils.toPem(csr); - SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument( - "signature", - 0, - new VespaUniqueInstanceId(0, "cluster", "instance", "application", "tenant", "region", "prod", IdentityType.NODE), - new AthenzService("domain", "service"), - 0, - "configserverhostname", - "instancehostname", - Instant.now().truncatedTo(ChronoUnit.MICROS), // Truncate to the precision given from EntityBindingsMapper.toAttestationData() - Collections.emptySet(), - IdentityType.NODE, - ClusterType.CONTAINER); - - var json = String.format("{\n" + - " \"provider\": \"provider_prod_us-north-1\",\n" + - " \"domain\": \"vespa.external\",\n" + - " \"service\": \"tenant\",\n" + - " \"attestationData\":\"%s\",\n" + - " \"csr\": \"" + csrPem + "\"\n" + - "}", StringUtilities.escape(EntityBindingsMapper.toAttestationData(signedIdentityDocument))); - var instanceRegistration = new InstanceRegistration("provider_prod_us-north-1", "vespa.external", - "tenant", signedIdentityDocument, - csr); - var deserialized = InstanceSerializer.registrationFromSlime(SlimeUtils.jsonToSlime(json)); - assertEquals(instanceRegistration, deserialized); - } - - @Test - void serialize_instance_identity() { - var certificate = CertificateTester.createCertificate(); - var pem = X509CertificateUtils.toPem(certificate); - var identity = new InstanceIdentity("provider_prod_us-north-1", "tenant", "node1.example.com", - Optional.of(certificate)); - var json = "{" + - "\"provider\":\"provider_prod_us-north-1\"," + - "\"service\":\"tenant\"," + - "\"instanceId\":\"node1.example.com\"," + - "\"x509Certificate\":\"" + pem.replace("\n", "\\n") + "\"" + - "}"; - assertEquals(json, asJsonString(InstanceSerializer.identityToSlime(identity))); - } - - @Test - void serialize_instance_refresh() { - var csr = CertificateTester.createCsr(); - var csrPem = Pkcs10CsrUtils.toPem(csr); - var json = "{\"csr\": \"" + csrPem + "\"}"; - var instanceRefresh = new InstanceRefresh(csr); - var deserialized = InstanceSerializer.refreshFromSlime(SlimeUtils.jsonToSlime(json)); - assertEquals(instanceRefresh, deserialized); - } - - private static String asJsonString(Slime slime) { - try { - return new String(SlimeUtils.toJsonBytes(slime), StandardCharsets.UTF_8); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/InstanceValidatorMock.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/InstanceValidatorMock.java deleted file mode 100644 index 4151c1f15d7..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/InstanceValidatorMock.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.restapi.mock; - -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceConfirmation; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceValidator; - -/** - * @author mortent - */ -public class InstanceValidatorMock extends InstanceValidator { - - public InstanceValidatorMock() { - super(null, null, null, null, null); - } - - @Override - public boolean isValidInstance(InstanceConfirmation instanceConfirmation) { - return instanceConfirmation.attributes.get(SAN_DNS_ATTRNAME) != null && - instanceConfirmation.attributes.get(SAN_IPS_ATTRNAME) != null; - } - - @Override - public boolean isValidRefresh(InstanceConfirmation confirmation) { - return confirmation.attributes.get(SAN_DNS_ATTRNAME) != null && - confirmation.attributes.get(SAN_IPS_ATTRNAME) != null; - } -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/PrincipalFromHeaderFilter.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/PrincipalFromHeaderFilter.java deleted file mode 100644 index df98ba75dd2..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/PrincipalFromHeaderFilter.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.restapi.mock; - -import com.yahoo.jdisc.handler.ResponseHandler; -import com.yahoo.jdisc.http.filter.DiscFilterRequest; -import com.yahoo.jdisc.http.filter.SecurityRequestFilter; -import com.yahoo.jdisc.http.server.jetty.RequestUtils; -import com.yahoo.security.X509CertificateUtils; -import com.yahoo.text.StringUtilities; -import com.yahoo.vespa.athenz.api.AthenzPrincipal; -import com.yahoo.vespa.athenz.api.AthenzService; - -import java.security.cert.X509Certificate; -import java.util.Optional; - -/** - * Read principal from http header - * - * @author mortent - */ -public class PrincipalFromHeaderFilter implements SecurityRequestFilter { - - @Override - public void filter(DiscFilterRequest request, ResponseHandler handler) { - String principal = request.getHeader("PRINCIPAL"); - request.setUserPrincipal(new AthenzPrincipal(new AthenzService(principal))); - - Optional<String> certificate = Optional.ofNullable(request.getHeader("CERTIFICATE")); - certificate.ifPresent(cert -> { - var x509cert = X509CertificateUtils.fromPem(StringUtilities.unescape(cert)); - request.setAttribute(RequestUtils.JDISC_REQUEST_X509CERT, new X509Certificate[]{x509cert}); - }); - } -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/SecretStoreMock.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/SecretStoreMock.java deleted file mode 100644 index 5a9f4fd0b76..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/SecretStoreMock.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.restapi.mock; - -import com.yahoo.component.AbstractComponent; -import com.yahoo.container.jdisc.secretstore.SecretStore; - -import java.util.HashMap; -import java.util.Map; - -/** - * @author mpolden - */ -public class SecretStoreMock extends AbstractComponent implements SecretStore { - - private final Map<String, String> secrets = new HashMap<>(); - - public SecretStoreMock setSecret(String key, String value) { - secrets.put(key, value); - return this; - } - - @Override - public String getSecret(String key) { - if (!secrets.containsKey(key)) throw new RuntimeException("No such key '" + key + "'"); - return secrets.get(key); - } - - @Override - public String getSecret(String key, int version) { - if (!secrets.containsKey(key)) throw new RuntimeException("No such key '" + key + "'"); - return secrets.get(key); - } - -} diff --git a/bootstrap.sh b/bootstrap.sh index e8730303ef7..0108999b1c6 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -42,7 +42,7 @@ echo "Using maven command: ${MAVEN_CMD}" echo "Using maven extra opts: ${MAVEN_EXTRA_OPTS}" mvn_install() { - ${MAVEN_CMD} --no-snapshot-updates -Dmaven.wagon.http.retryHandler.count=5 clean install ${MAVEN_EXTRA_OPTS} "$@" + ${MAVEN_CMD} --batch-mode --no-snapshot-updates -Dmaven.wagon.http.retryHandler.count=5 clean install ${MAVEN_EXTRA_OPTS} "$@" } # Generate vtag map diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java index 30810d428c1..29a54548256 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java @@ -26,25 +26,26 @@ public class AutoscalingMetrics { // Memory util metrics.add(HostedNodeAdminMetrics.MEM_UTIL.baseName()); // node level - default - metrics.add(SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_MEMORY.average()); // better for content as it is the basis for blocking + metrics.add(SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_MEMORY.average()); // the basis for blocking // Disk util metrics.add(HostedNodeAdminMetrics.DISK_UTIL.baseName()); // node level -default - metrics.add(SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_DISK.average()); // better for content as it is the basis for blocking + metrics.add(SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_DISK.average()); // the basis for blocking - metrics.add(ContainerMetrics.APPLICATION_GENERATION.baseName()); + metrics.add(ContainerMetrics.APPLICATION_GENERATION.last()); + metrics.add(SearchNodeMetrics.CONTENT_PROTON_CONFIG_GENERATION.last()); - metrics.add(ContainerMetrics.IN_SERVICE.baseName()); + metrics.add(ContainerMetrics.IN_SERVICE.last()); // Query rate - metrics.add(ContainerMetrics.QUERIES.rate()); // container - metrics.add(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_QUERIES.rate()); // content + metrics.add(ContainerMetrics.QUERIES.rate()); + metrics.add(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_QUERIES.rate()); // Write rate - metrics.add(ContainerMetrics.FEED_HTTP_REQUESTS.rate()); // container - metrics.add(StorageMetrics.VDS_FILESTOR_ALLTHREADS_PUT_COUNT.rate()); // content - metrics.add(StorageMetrics.VDS_FILESTOR_ALLTHREADS_REMOVE_COUNT.rate()); // content - metrics.add(StorageMetrics.VDS_FILESTOR_ALLTHREADS_UPDATE_COUNT.rate()); // content + metrics.add(ContainerMetrics.FEED_HTTP_REQUESTS.rate()); + metrics.add(StorageMetrics.VDS_FILESTOR_ALLTHREADS_PUT_COUNT.rate()); + metrics.add(StorageMetrics.VDS_FILESTOR_ALLTHREADS_REMOVE_COUNT.rate()); + metrics.add(StorageMetrics.VDS_FILESTOR_ALLTHREADS_UPDATE_COUNT.rate()); return new MetricSet("autoscaling", toMetrics(metrics)); } diff --git a/configdefinitions/src/vespa/athenz-provider-service.def b/configdefinitions/src/vespa/athenz-provider-service.def index 2131aa88d30..4c9c74f9b8f 100644 --- a/configdefinitions/src/vespa/athenz-provider-service.def +++ b/configdefinitions/src/vespa/athenz-provider-service.def @@ -13,6 +13,11 @@ secretName string # Secret version secretVersion int +# Tempory resources +sisSecretName string default="" +sisSecretVersion int default=0 +sisUrl string default = "" + # Secret name of CA certificate caCertSecretName string diff --git a/configserver/CMakeLists.txt b/configserver/CMakeLists.txt index f189dc4f2c1..201d419c669 100644 --- a/configserver/CMakeLists.txt +++ b/configserver/CMakeLists.txt @@ -12,7 +12,6 @@ install(DIRECTORY DESTINATION conf/configserver) install(DIRECTORY DESTINATION conf/configserver-app/components) install(DIRECTORY DESTINATION conf/configserver-app/config-models) -install_symlink(lib/jars/athenz-identity-provider-service-jar-with-dependencies.jar conf/configserver-app/components/athenz-identity-provider-service.jar) install_symlink(lib/jars/config-model-fat.jar conf/configserver-app/components/config-model-fat.jar) install_symlink(lib/jars/configserver-flags-jar-with-dependencies.jar conf/configserver-app/components/configserver-flags.jar) install_symlink(lib/jars/flags-jar-with-dependencies.jar conf/configserver-app/components/flags.jar) diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java index 2238abc584e..b9a6d8d9462 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java @@ -6,6 +6,7 @@ import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.core.VipStatusConfig; import com.yahoo.container.jdisc.state.StateMonitor; import com.yahoo.jdisc.Metric; +import com.yahoo.metrics.ContainerMetrics; import java.util.Map; import java.util.stream.Collectors; @@ -119,7 +120,7 @@ public class VipStatus { else if (healthState.status() == StateMonitor.Status.up) healthState.status(StateMonitor.Status.down); - metric.set("in_service", currentlyInRotation ? 1 : 0, metric.createContext(Map.of())); + metric.set(ContainerMetrics.IN_SERVICE.baseName(), currentlyInRotation ? 1 : 0, metric.createContext(Map.of())); } } diff --git a/container-core/src/main/java/com/yahoo/metrics/SearchNodeMetrics.java b/container-core/src/main/java/com/yahoo/metrics/SearchNodeMetrics.java index ed38a4b2ba3..c39054c878c 100644 --- a/container-core/src/main/java/com/yahoo/metrics/SearchNodeMetrics.java +++ b/container-core/src/main/java/com/yahoo/metrics/SearchNodeMetrics.java @@ -17,7 +17,11 @@ public enum SearchNodeMetrics implements VespaMetrics { CONTENT_PROTON_DOCUMENTDB_INDEX_DOCS_IN_MEMORY("content.proton.documentdb.index.docs_in_memory", Unit.DOCUMENT, "Number of documents in memory index"), CONTENT_PROTON_DOCUMENTDB_DISK_USAGE("content.proton.documentdb.disk_usage", Unit.BYTE, "The total disk usage (in bytes) for this document db"), CONTENT_PROTON_DOCUMENTDB_MEMORY_USAGE_ALLOCATED_BYTES("content.proton.documentdb.memory_usage.allocated_bytes", Unit.BYTE, "The number of allocated bytes"), + CONTENT_PROTON_DOCUMENTDB_MEMORY_USAGE_DEAD_BYTES("content.proton.documentdb.memory_usage.dead_bytes", Unit.BYTE, "The number of dead bytes (<= used_bytes)"), + CONTENT_PROTON_DOCUMENTDB_MEMORY_USAGE_ONHOLD_BYTES("content.proton.documentdb.memory_usage.onhold_bytes", Unit.BYTE, "The number of bytes on hold"), + CONTENT_PROTON_DOCUMENTDB_MEMORY_USAGE_USED_BYTES("content.proton.documentdb.memory_usage.used_bytes", Unit.BYTE, "The number of used bytes (<= allocated_bytes)"), CONTENT_PROTON_DOCUMENTDB_HEART_BEAT_AGE("content.proton.documentdb.heart_beat_age", Unit.SECOND, "How long ago (in seconds) heart beat maintenace job was run"), + CONTENT_PROTON_DOCSUM_COUNT("content.proton.docsum.count", Unit.REQUEST, "Docsum requests handled"), CONTENT_PROTON_DOCSUM_DOCS("content.proton.docsum.docs", Unit.DOCUMENT, "Total docsums returned"), CONTENT_PROTON_DOCSUM_LATENCY("content.proton.docsum.latency", Unit.MILLISECOND, "Docsum request latency"), @@ -35,30 +39,37 @@ public enum SearchNodeMetrics implements VespaMetrics { CONTENT_PROTON_EXECUTOR_PROTON_ACCEPTED("content.proton.executor.proton.accepted", Unit.TASK, "Number of executor proton accepted tasks"), CONTENT_PROTON_EXECUTOR_PROTON_WAKEUPS("content.proton.executor.proton.wakeups", Unit.WAKEUP, "Number of times a executor proton worker thread has been woken up"), CONTENT_PROTON_EXECUTOR_PROTON_UTILIZATION("content.proton.executor.proton.utilization", Unit.FRACTION, "Ratio of time the executor proton worker threads has been active"), + CONTENT_PROTON_EXECUTOR_PROTON_REJECTED("content.proton.executor.proton.rejected", Unit.TASK, "Number of rejected tasks"), CONTENT_PROTON_EXECUTOR_FLUSH_QUEUESIZE("content.proton.executor.flush.queuesize", Unit.TASK, "Size of executor flush task queue"), CONTENT_PROTON_EXECUTOR_FLUSH_ACCEPTED("content.proton.executor.flush.accepted", Unit.TASK, "Number of accepted executor flush tasks"), CONTENT_PROTON_EXECUTOR_FLUSH_WAKEUPS("content.proton.executor.flush.wakeups", Unit.WAKEUP, "Number of times a executor flush worker thread has been woken up"), CONTENT_PROTON_EXECUTOR_FLUSH_UTILIZATION("content.proton.executor.flush.utilization", Unit.FRACTION, "Ratio of time the executor flush worker threads has been active"), + CONTENT_PROTON_EXECUTOR_FLUSH_REJECTED("content.proton.executor.flush.rejected", Unit.TASK, "Number of rejected tasks"), CONTENT_PROTON_EXECUTOR_MATCH_QUEUESIZE("content.proton.executor.match.queuesize", Unit.TASK, "Size of executor match task queue"), CONTENT_PROTON_EXECUTOR_MATCH_ACCEPTED("content.proton.executor.match.accepted", Unit.TASK, "Number of accepted executor match tasks"), CONTENT_PROTON_EXECUTOR_MATCH_WAKEUPS("content.proton.executor.match.wakeups", Unit.WAKEUP, "Number of times a executor match worker thread has been woken up"), CONTENT_PROTON_EXECUTOR_MATCH_UTILIZATION("content.proton.executor.match.utilization", Unit.FRACTION, "Ratio of time the executor match worker threads has been active"), + CONTENT_PROTON_EXECUTOR_MATCH_REJECTED("content.proton.executor.match.rejected", Unit.TASK, "Number of rejected tasks"), CONTENT_PROTON_EXECUTOR_DOCSUM_QUEUESIZE("content.proton.executor.docsum.queuesize", Unit.TASK, "Size of executor docsum task queue"), CONTENT_PROTON_EXECUTOR_DOCSUM_ACCEPTED("content.proton.executor.docsum.accepted", Unit.TASK, "Number of executor accepted docsum tasks"), CONTENT_PROTON_EXECUTOR_DOCSUM_WAKEUPS("content.proton.executor.docsum.wakeups", Unit.WAKEUP, "Number of times a executor docsum worker thread has been woken up"), CONTENT_PROTON_EXECUTOR_DOCSUM_UTILIZATION("content.proton.executor.docsum.utilization", Unit.FRACTION, "Ratio of time the executor docsum worker threads has been active"), + CONTENT_PROTON_EXECUTOR_DOCSUM_REJECTED("content.proton.executor.docsum.rejected", Unit.TASK, "Number of rejected tasks"), CONTENT_PROTON_EXECUTOR_SHARED_QUEUESIZE("content.proton.executor.shared.queuesize", Unit.TASK, "Size of executor shared task queue"), CONTENT_PROTON_EXECUTOR_SHARED_ACCEPTED("content.proton.executor.shared.accepted", Unit.TASK, "Number of executor shared accepted tasks"), CONTENT_PROTON_EXECUTOR_SHARED_WAKEUPS("content.proton.executor.shared.wakeups", Unit.WAKEUP, "Number of times a executor shared worker thread has been woken up"), CONTENT_PROTON_EXECUTOR_SHARED_UTILIZATION("content.proton.executor.shared.utilization", Unit.FRACTION, "Ratio of time the executor shared worker threads has been active"), + CONTENT_PROTON_EXECUTOR_SHARED_REJECTED("content.proton.executor.shared.rejected", Unit.TASK, "Number of rejected tasks"), CONTENT_PROTON_EXECUTOR_WARMUP_QUEUESIZE("content.proton.executor.warmup.queuesize", Unit.TASK, "Size of executor warmup task queue"), CONTENT_PROTON_EXECUTOR_WARMUP_ACCEPTED("content.proton.executor.warmup.accepted", Unit.TASK, "Number of accepted executor warmup tasks"), CONTENT_PROTON_EXECUTOR_WARMUP_WAKEUPS("content.proton.executor.warmup.wakeups", Unit.WAKEUP, "Number of times a warmup executor worker thread has been woken up"), CONTENT_PROTON_EXECUTOR_WARMUP_UTILIZATION("content.proton.executor.warmup.utilization", Unit.FRACTION, "Ratio of time the executor warmup worker threads has been active"), + CONTENT_PROTON_EXECUTOR_WARMUP_REJECTED("content.proton.executor.warmup.rejected", Unit.TASK, "Number of rejected tasks"), CONTENT_PROTON_EXECUTOR_FIELD_WRITER_QUEUESIZE("content.proton.executor.field_writer.queuesize", Unit.TASK, "Size of executor field writer task queue"), CONTENT_PROTON_EXECUTOR_FIELD_WRITER_ACCEPTED("content.proton.executor.field_writer.accepted", Unit.TASK, "Number of accepted executor field writer tasks"), CONTENT_PROTON_EXECUTOR_FIELD_WRITER_WAKEUPS("content.proton.executor.field_writer.wakeups", Unit.WAKEUP, "Number of times a executor field writer worker thread has been woken up"), CONTENT_PROTON_EXECUTOR_FIELD_WRITER_UTILIZATION("content.proton.executor.field_writer.utilization", Unit.FRACTION, "Ratio of time the executor fieldwriter worker threads has been active"), + CONTENT_PROTON_EXECUTOR_FIELD_WRITER_REJECTED("content.proton.executor.field_writer.rejected", Unit.TASK, "Number of rejected tasks"), // jobs CONTENT_PROTON_DOCUMENTDB_JOB_TOTAL("content.proton.documentdb.job.total", Unit.FRACTION, "The job load average total of all job metrics"), @@ -76,14 +87,32 @@ public enum SearchNodeMetrics implements VespaMetrics { CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_MASTER_ACCEPTED("content.proton.documentdb.threading_service.master.accepted", Unit.TASK, "Number of accepted threading service master tasks"), CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_MASTER_WAKEUPS("content.proton.documentdb.threading_service.master.wakeups", Unit.WAKEUP, "Number of times a threading service master worker thread has been woken up"), CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_MASTER_UTILIZATION("content.proton.documentdb.threading_service.master.utilization", Unit.FRACTION, "Ratio of time the threading service master worker threads has been active"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_MASTER_REJECTED("content.proton.documentdb.threading_service.master.rejected", Unit.TASK, "Number of rejected tasks"), CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_QUEUESIZE("content.proton.documentdb.threading_service.index.queuesize", Unit.TASK, "Size of threading service index task queue"), CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_ACCEPTED("content.proton.documentdb.threading_service.index.accepted", Unit.TASK, "Number of accepted threading service index tasks"), CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_WAKEUPS("content.proton.documentdb.threading_service.index.wakeups", Unit.WAKEUP, "Number of times a threading service index worker thread has been woken up"), CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_UTILIZATION("content.proton.documentdb.threading_service.index.utilization", Unit.FRACTION, "Ratio of time the threading service index worker threads has been active"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_REJECTED("content.proton.documentdb.threading_service.index.rejected", Unit.TASK, "Number of rejected tasks"), CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_QUEUESIZE("content.proton.documentdb.threading_service.summary.queuesize", Unit.TASK, "Size of threading service summary task queue"), CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_ACCEPTED("content.proton.documentdb.threading_service.summary.accepted", Unit.TASK, "Number of accepted threading service summary tasks"), CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_WAKEUPS("content.proton.documentdb.threading_service.summary.wakeups", Unit.WAKEUP, "Number of times a threading service summary worker thread has been woken up"), CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_UTILIZATION("content.proton.documentdb.threading_service.summary.utilization", Unit.FRACTION, "Ratio of time the threading service summary worker threads has been active"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_REJECTED("content.proton.documentdb.threading_service.summary.rejected", Unit.TASK, "Number of rejected tasks"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_ATTRIBUTE_FIELD_WRITER_ACCEPTED("content.proton.documentdb.threading_service.attribute_field_writer.accepted", Unit.TASK, "Number of accepted tasks"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_ATTRIBUTE_FIELD_WRITER_QUEUESIZE("content.proton.documentdb.threading_service.attribute_field_writer.queuesize", Unit.TASK, "Size of task queue"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_ATTRIBUTE_FIELD_WRITER_REJECTED("content.proton.documentdb.threading_service.attribute_field_writer.rejected", Unit.TASK, "Number of rejected tasks"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_ATTRIBUTE_FIELD_WRITER_UTILIZATION("content.proton.documentdb.threading_service.attribute_field_writer.utilization", Unit.FRACTION, "Ratio of time the worker threads has been active"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_ATTRIBUTE_FIELD_WRITER_WAKEUPS("content.proton.documentdb.threading_service.attribute_field_writer.wakeups", Unit.WAKEUP, "Number of times a worker thread has been woken up"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_INVERTER_ACCEPTED("content.proton.documentdb.threading_service.index_field_inverter.accepted", Unit.TASK, "Number of accepted tasks"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_INVERTER_QUEUESIZE("content.proton.documentdb.threading_service.index_field_inverter.queuesize", Unit.TASK, "Size of task queue"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_INVERTER_REJECTED("content.proton.documentdb.threading_service.index_field_inverter.rejected", Unit.TASK, "Number of rejected tasks"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_INVERTER_UTILIZATION("content.proton.documentdb.threading_service.index_field_inverter.utilization", Unit.FRACTION, "Ratio of time the worker threads has been active"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_INVERTER_WAKEUPS("content.proton.documentdb.threading_service.index_field_inverter.wakeups", Unit.WAKEUP, "Number of times a worker thread has been woken up"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_WRITER_ACCEPTED("content.proton.documentdb.threading_service.index_field_writer.accepted", Unit.TASK, "Number of accepted tasks"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_WRITER_QUEUESIZE("content.proton.documentdb.threading_service.index_field_writer.queuesize", Unit.TASK, "Size of task queue"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_WRITER_REJECTED("content.proton.documentdb.threading_service.index_field_writer.rejected", Unit.TASK, "Number of rejected tasks"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_WRITER_UTILIZATION("content.proton.documentdb.threading_service.index_field_writer.utilization", Unit.FRACTION, "Ratio of time the worker threads has been active"), + CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_WRITER_WAKEUPS("content.proton.documentdb.threading_service.index_field_writer.wakeups", Unit.WAKEUP, "Number of times a worker thread has been woken up"), // lid space CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LID_BLOAT_FACTOR("content.proton.documentdb.ready.lid_space.lid_bloat_factor", Unit.FRACTION, "The bloat factor of this lid space, indicating the total amount of holes in the allocated lid space ((lid_limit - used_lids) / lid_limit)"), @@ -91,16 +120,19 @@ public enum SearchNodeMetrics implements VespaMetrics { CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LID_LIMIT("content.proton.documentdb.ready.lid_space.lid_limit", Unit.DOCUMENTID, "The size of the allocated lid space"), CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_HIGHEST_USED_LID("content.proton.documentdb.ready.lid_space.highest_used_lid", Unit.DOCUMENTID, "The highest used lid"), CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_USED_LIDS("content.proton.documentdb.ready.lid_space.used_lids", Unit.DOCUMENTID, "The number of lids used"), + CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LOWEST_FREE_LID("content.proton.documentdb.ready.lid_space.lowest_free_lid", Unit.DOCUMENTID, "The lowest free local document id"), CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_LID_BLOAT_FACTOR("content.proton.documentdb.notready.lid_space.lid_bloat_factor", Unit.FRACTION, "The bloat factor of this lid space, indicating the total amount of holes in the allocated lid space ((lid_limit - used_lids) / lid_limit)"), CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_LID_FRAGMENTATION_FACTOR("content.proton.documentdb.notready.lid_space.lid_fragmentation_factor", Unit.FRACTION, "The fragmentation factor of this lid space, indicating the amount of holes in the currently used part of the lid space ((highest_used_lid - used_lids) / highest_used_lid)"), CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_LID_LIMIT("content.proton.documentdb.notready.lid_space.lid_limit", Unit.DOCUMENTID, "The size of the allocated lid space"), CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_HIGHEST_USED_LID("content.proton.documentdb.notready.lid_space.highest_used_lid", Unit.DOCUMENTID, "The highest used lid"), CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_USED_LIDS("content.proton.documentdb.notready.lid_space.used_lids", Unit.DOCUMENTID, "The number of lids used"), + CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_LOWEST_FREE_LID("content.proton.documentdb.notready.lid_space.lowest_free_lid", Unit.DOCUMENTID, "The lowest free local document id"), CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_LID_BLOAT_FACTOR("content.proton.documentdb.removed.lid_space.lid_bloat_factor", Unit.FRACTION, "The bloat factor of this lid space, indicating the total amount of holes in the allocated lid space ((lid_limit - used_lids) / lid_limit)"), CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_LID_FRAGMENTATION_FACTOR("content.proton.documentdb.removed.lid_space.lid_fragmentation_factor", Unit.FRACTION, "The fragmentation factor of this lid space, indicating the amount of holes in the currently used part of the lid space ((highest_used_lid - used_lids) / highest_used_lid)"), CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_LID_LIMIT("content.proton.documentdb.removed.lid_space.lid_limit", Unit.DOCUMENTID, "The size of the allocated lid space"), CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_HIGHEST_USED_LID("content.proton.documentdb.removed.lid_space.highest_used_lid", Unit.DOCUMENTID, "The highest used lid"), CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_USED_LIDS("content.proton.documentdb.removed.lid_space.used_lids", Unit.DOCUMENTID, "The number of lids used"), + CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_LOWEST_FREE_LID("content.proton.documentdb.removed.lid_space.lowest_free_lid", Unit.DOCUMENTID, "The lowest free local document id"), // bucket move CONTENT_PROTON_DOCUMENTDB_BUCKET_MOVE_BUCKETS_PENDING("content.proton.documentdb.bucket_move.buckets_pending", Unit.BUCKET, "The number of buckets left to move"), @@ -120,6 +152,10 @@ public enum SearchNodeMetrics implements VespaMetrics { CONTENT_PROTON_RESOURCE_USAGE_MALLOC_ARENA("content.proton.resource_usage.malloc_arena", Unit.BYTE, "Size of malloc arena"), CONTENT_PROTON_DOCUMENTDB_ATTRIBUTE_RESOURCE_USAGE_ADDRESS_SPACE("content.proton.documentdb.attribute.resource_usage.address_space", Unit.FRACTION, "The max relative address space used among components in all attribute vectors in this document db (value in the range [0, 1])"), CONTENT_PROTON_DOCUMENTDB_ATTRIBUTE_RESOURCE_USAGE_FEEDING_BLOCKED("content.proton.documentdb.attribute.resource_usage.feeding_blocked", Unit.BINARY, "Whether feeding is blocked due to attribute resource limits being reached (value is either 0 or 1)"), + CONTENT_PROTON_DOCUMENTDB_ATTRIBUTE_MEMORY_USAGE_ALLOCATED_BYTES("content.proton.documentdb.attribute.memory_usage.allocated_bytes", Unit.BYTE, "The number of allocated bytes"), + CONTENT_PROTON_DOCUMENTDB_ATTRIBUTE_MEMORY_USAGE_DEAD_BYTES("content.proton.documentdb.attribute.memory_usage.dead_bytes", Unit.BYTE, "The number of dead bytes (<= used_bytes)"), + CONTENT_PROTON_DOCUMENTDB_ATTRIBUTE_MEMORY_USAGE_ONHOLD_BYTES("content.proton.documentdb.attribute.memory_usage.onhold_bytes", Unit.BYTE, "The number of bytes on hold"), + CONTENT_PROTON_DOCUMENTDB_ATTRIBUTE_MEMORY_USAGE_USED_BYTES("content.proton.documentdb.attribute.memory_usage.used_bytes", Unit.BYTE, "The number of used bytes (<= allocated_bytes)"), // CPU util CONTENT_PROTON_RESOURCE_USAGE_CPU_UTIL_SETUP("content.proton.resource_usage.cpu_util.setup", Unit.FRACTION, "cpu used by system init and (re-)configuration"), @@ -157,14 +193,21 @@ public enum SearchNodeMetrics implements VespaMetrics { CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_MEMORY_USAGE_ONHOLD_BYTES("content.proton.documentdb.removed.document_store.memory_usage.onhold_bytes", Unit.BYTE, "The number of bytes on hold"), // document store cache + CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_CACHE_ELEMENTS("content.proton.documentdb.ready.document_store.cache.elements", Unit.ITEM, "Number of elements in the cache"), CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_CACHE_MEMORY_USAGE("content.proton.documentdb.ready.document_store.cache.memory_usage", Unit.BYTE, "Memory usage of the cache (in bytes)"), CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_CACHE_HIT_RATE("content.proton.documentdb.ready.document_store.cache.hit_rate", Unit.FRACTION, "Rate of hits in the cache compared to number of lookups"), CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_CACHE_LOOKUPS("content.proton.documentdb.ready.document_store.cache.lookups", Unit.OPERATION, "Number of lookups in the cache (hits + misses)"), CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_CACHE_INVALIDATIONS("content.proton.documentdb.ready.document_store.cache.invalidations", Unit.OPERATION, "Number of invalidations (erased elements) in the cache. "), + CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_CACHE_ELEMENTS("content.proton.documentdb.notready.document_store.cache.elements", Unit.ITEM, "Number of elements in the cache"), CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_CACHE_MEMORY_USAGE("content.proton.documentdb.notready.document_store.cache.memory_usage", Unit.BYTE, "Memory usage of the cache (in bytes)"), CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_CACHE_HIT_RATE("content.proton.documentdb.notready.document_store.cache.hit_rate", Unit.FRACTION, "Rate of hits in the cache compared to number of lookups"), CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_CACHE_LOOKUPS("content.proton.documentdb.notready.document_store.cache.lookups", Unit.OPERATION, "Number of lookups in the cache (hits + misses)"), CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_CACHE_INVALIDATIONS("content.proton.documentdb.notready.document_store.cache.invalidations", Unit.OPERATION, "Number of invalidations (erased elements) in the cache. "), + CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_CACHE_ELEMENTS("content.proton.documentdb.removed.document_store.cache.elements", Unit.ITEM, "Number of elements in the cache"), + CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_CACHE_HIT_RATE("content.proton.documentdb.removed.document_store.cache.hit_rate", Unit.FRACTION, "Rate of hits in the cache compared to number of lookups"), + CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_CACHE_INVALIDATIONS("content.proton.documentdb.removed.document_store.cache.invalidations", Unit.ITEM, "Number of invalidations (erased elements) in the cache. "), + CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_CACHE_LOOKUPS("content.proton.documentdb.removed.document_store.cache.lookups", Unit.OPERATION, "Number of lookups in the cache (hits + misses)"), + CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_CACHE_MEMORY_USAGE("content.proton.documentdb.removed.document_store.cache.memory_usage", Unit.BYTE, "Memory usage of the cache (in bytes)"), // attribute CONTENT_PROTON_DOCUMENTDB_READY_ATTRIBUTE_MEMORY_USAGE_ALLOCATED_BYTES("content.proton.documentdb.ready.attribute.memory_usage.allocated_bytes", Unit.BYTE, "The number of allocated bytes"), @@ -181,6 +224,7 @@ public enum SearchNodeMetrics implements VespaMetrics { CONTENT_PROTON_DOCUMENTDB_INDEX_MEMORY_USAGE_USED_BYTES("content.proton.documentdb.index.memory_usage.used_bytes", Unit.BYTE, "The number of used bytes (<= allocated_bytes)"), CONTENT_PROTON_DOCUMENTDB_INDEX_MEMORY_USAGE_DEAD_BYTES("content.proton.documentdb.index.memory_usage.dead_bytes", Unit.BYTE, "The number of dead bytes (<= used_bytes)"), CONTENT_PROTON_DOCUMENTDB_INDEX_MEMORY_USAGE_ONHOLD_BYTES("content.proton.documentdb.index.memory_usage.onhold_bytes", Unit.BYTE, "The number of bytes on hold"), + CONTENT_PROTON_DOCUMENTDB_INDEX_DISK_USAGE("content.proton.documentdb.index.disk_usage", Unit.BYTE, "Disk space usage in bytes"), // matching CONTENT_PROTON_DOCUMENTDB_MATCHING_QUERIES("content.proton.documentdb.matching.queries", Unit.QUERY, "Number of queries executed"), @@ -188,6 +232,7 @@ public enum SearchNodeMetrics implements VespaMetrics { CONTENT_PROTON_DOCUMENTDB_MATCHING_QUERY_LATENCY("content.proton.documentdb.matching.query_latency", Unit.SECOND, "Total average latency (sec) when matching and ranking a query"), CONTENT_PROTON_DOCUMENTDB_MATCHING_QUERY_SETUP_TIME("content.proton.documentdb.matching.query_setup_time", Unit.SECOND, "Average time (sec) spent setting up and tearing down queries"), CONTENT_PROTON_DOCUMENTDB_MATCHING_DOCS_MATCHED("content.proton.documentdb.matching.docs_matched", Unit.DOCUMENT, "Number of documents matched"), + CONTENT_PROTON_DOCUMENTDB_MATCHING_DOCS_RANKED("content.proton.documentdb.matching.docs_ranked", Unit.DOCUMENT, "Number of documents ranked (first phase)"), CONTENT_PROTON_DOCUMENTDB_MATCHING_DOCS_RERANKED("content.proton.documentdb.matching.docs_reranked", Unit.DOCUMENT, "Number of documents re-ranked (second phase)"), CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_QUERIES("content.proton.documentdb.matching.rank_profile.queries", Unit.QUERY, "Number of queries executed"), CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_SOFT_DOOMED_QUERIES("content.proton.documentdb.matching.rank_profile.soft_doomed_queries", Unit.QUERY, "Number of queries hitting the soft timeout"), @@ -197,11 +242,39 @@ public enum SearchNodeMetrics implements VespaMetrics { CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_GROUPING_TIME("content.proton.documentdb.matching.rank_profile.grouping_time", Unit.SECOND, "Average time (sec) spent on grouping"), CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_RERANK_TIME("content.proton.documentdb.matching.rank_profile.rerank_time", Unit.SECOND, "Average time (sec) spent on 2nd phase ranking"), CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_DOCS_MATCHED("content.proton.documentdb.matching.rank_profile.docs_matched", Unit.DOCUMENT, "Number of documents matched"), + CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_DOCS_RANKED("content.proton.documentdb.matching.rank_profile.docs_ranked", Unit.DOCUMENT, "Number of documents ranked (first phase)"), + CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_DOCS_RERANKED("content.proton.documentdb.matching.rank_profile.docs_reranked", Unit.DOCUMENT, "Number of documents re-ranked (second phase)"), CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_LIMITED_QUERIES("content.proton.documentdb.matching.rank_profile.limited_queries", Unit.QUERY, "Number of queries limited in match phase"), + CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_DOCID_PARTITION_ACTIVE_TIME("content.proton.documentdb.matching.rank_profile.docid_partition.active_time", Unit.SECOND, "Time (sec) spent doing actual work"), + CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_DOCID_PARTITION_DOCS_MATCHED("content.proton.documentdb.matching.rank_profile.docid_partition.docs_matched", Unit.DOCUMENT, "Number of documents matched"), + CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_DOCID_PARTITION_DOCS_RANKED("content.proton.documentdb.matching.rank_profile.docid_partition.docs_ranked", Unit.DOCUMENT, "Number of documents ranked (first phase)"), + CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_DOCID_PARTITION_DOCS_RERANKED("content.proton.documentdb.matching.rank_profile.docid_partition.docs_reranked", Unit.DOCUMENT, "Number of documents re-ranked (second phase)"), + CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_DOCID_PARTITION_WAIT_TIME("content.proton.documentdb.matching.rank_profile.docid_partition.wait_time", Unit.SECOND, "Time (sec) spent waiting for other external threads and resources"), + CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_MATCH_TIME("content.proton.documentdb.matching.rank_profile.match_time", Unit.SECOND, "Average time (sec) for matching a query (1st phase)"), // feeding CONTENT_PROTON_DOCUMENTDB_FEEDING_COMMIT_OPERATIONS("content.proton.documentdb.feeding.commit.operations", Unit.OPERATION, "Number of operations included in a commit"), - CONTENT_PROTON_DOCUMENTDB_FEEDING_COMMIT_LATENCY("content.proton.documentdb.feeding.commit.latency", Unit.SECOND, "Latency for commit in seconds"); + CONTENT_PROTON_DOCUMENTDB_FEEDING_COMMIT_LATENCY("content.proton.documentdb.feeding.commit.latency", Unit.SECOND, "Latency for commit in seconds"), + + + // Metrics emitters not used in any metrics sets + CONTENT_PROTON_SESSION_CACHE_GROUPING_NUM_CACHED("content.proton.session_cache.grouping.num_cached", Unit.SESSION, "Number of currently cached sessions"), + CONTENT_PROTON_SESSION_CACHE_GROUPING_NUM_DROPPED("content.proton.session_cache.grouping.num_dropped", Unit.SESSION, "Number of dropped cached sessions"), + CONTENT_PROTON_SESSION_CACHE_GROUPING_NUM_INSERT("content.proton.session_cache.grouping.num_insert", Unit.SESSION, "Number of inserted sessions"), + CONTENT_PROTON_SESSION_CACHE_GROUPING_NUM_PICK("content.proton.session_cache.grouping.num_pick", Unit.SESSION, "Number if picked sessions"), + CONTENT_PROTON_SESSION_CACHE_GROUPING_NUM_TIMEDOUT("content.proton.session_cache.grouping.num_timedout", Unit.SESSION, "Number of timed out sessions"), + CONTENT_PROTON_SESSION_CACHE_SEARCH_NUM_CACHED("content.proton.session_cache.search.num_cached", Unit.SESSION, "Number of currently cached sessions"), + CONTENT_PROTON_SESSION_CACHE_SEARCH_NUM_DROPPED("content.proton.session_cache.search.num_dropped", Unit.SESSION, "Number of dropped cached sessions"), + CONTENT_PROTON_SESSION_CACHE_SEARCH_NUM_INSERT("content.proton.session_cache.search.num_insert", Unit.SESSION, "Number of inserted sessions"), + CONTENT_PROTON_SESSION_CACHE_SEARCH_NUM_PICK("content.proton.session_cache.search.num_pick", Unit.SESSION, "Number if picked sessions"), + CONTENT_PROTON_SESSION_CACHE_SEARCH_NUM_TIMEDOUT("content.proton.session_cache.search.num_timedout", Unit.SESSION, "Number of timed out sessions"), + + METRICMANAGER_PERIODICHOOKLATENCY("metricmanager.periodichooklatency", Unit.MILLISECOND, "Time in ms used to update a single periodic hook"), + METRICMANAGER_RESETLATENCY("metricmanager.resetlatency", Unit.MILLISECOND, "Time in ms used to reset all metrics."), + METRICMANAGER_SLEEPTIME("metricmanager.sleeptime", Unit.MILLISECOND, "Time in ms worker thread is sleeping"), + METRICMANAGER_SNAPSHOTHOOKLATENCY("metricmanager.snapshothooklatency", Unit.MILLISECOND, "Time in ms used to update a single snapshot hook"), + METRICMANAGER_SNAPSHOTLATENCY("metricmanager.snapshotlatency", Unit.MILLISECOND, "Time in ms used to take a snapshot"); + private final String name; private final Unit unit; diff --git a/dist/vespa.spec b/dist/vespa.spec index 5b3a172d015..fd2eda6e107 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -452,17 +452,34 @@ export FACTORY_VESPA_VERSION=%{version} mvn --batch-mode -e -N io.takari:maven:wrapper -Dmaven=3.6.3 %endif %{?_use_mvn_wrapper:env VESPA_MAVEN_COMMAND=$(pwd)/mvnw }sh bootstrap.sh java -%{?_use_mvn_wrapper:./mvnw}%{!?_use_mvn_wrapper:mvn} --batch-mode -nsu -T 1C install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true +%{?_use_mvn_wrapper:./mvnw}%{!?_use_mvn_wrapper:mvn} --batch-mode -nsu -T 1C install -DskipTests -Dmaven.javadoc.skip=true %{_command_cmake} -DCMAKE_INSTALL_PREFIX=%{_prefix} \ -DJAVA_HOME=$JAVA_HOME \ -DVESPA_USER=%{_vespa_user} \ -DVESPA_UNPRIVILEGED=no \ + %{_cmake_extra_opts} \ . make %{_smp_mflags} VERSION=%{version} CI=true make -C client/go install-all %endif +%check +%if ! 0%{?installdir:1} +%if 0%{?_java_home:1} +export JAVA_HOME=%{?_java_home} +%else +export JAVA_HOME=/usr/lib/jvm/java-17-openjdk +%endif +export PATH="$JAVA_HOME/bin:$PATH" +%if 0%{?el8} +python3.9 -m pip install --user pytest +%endif +export PYTHONPATH="$PYTHONPATH:/usr/local/lib/$(basename $(readlink -f $(which python3)))/site-packages" +#%{?_use_mvn_wrapper:./mvnw}%{!?_use_mvn_wrapper:mvn} --batch-mode -nsu -T 1C -Dmaven.javadoc.skip=true test +make test ARGS="--output-on-failure %{_smp_mflags}" +%endif + %install rm -rf %{buildroot} @@ -586,7 +603,6 @@ fi %{_prefix}/include %dir %{_prefix}/lib %dir %{_prefix}/lib/jars -%{_prefix}/lib/jars/athenz-identity-provider-service-jar-with-dependencies.jar %{_prefix}/lib/jars/cloud-tenant-cd-jar-with-dependencies.jar %{_prefix}/lib/jars/clustercontroller-apps-jar-with-dependencies.jar %{_prefix}/lib/jars/clustercontroller-core-jar-with-dependencies.jar diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index f4584f564ad..a7cf41f2462 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -46,6 +46,16 @@ public class Flags { private static volatile TreeMap<FlagId, FlagDefinition> flags = new TreeMap<>(); + public static final UnboundBooleanFlag DROP_CACHES = defineFeatureFlag( + "drop-caches", false, + List.of("hakonhall", "baldersheim"), "2023-03-06", "2023-04-05", + "Drop caches on tenant hosts", + "Takes effect on next tick", + ZONE_ID, + // The application ID is the exclusive application ID associated with the host, + // if any, or otherwise hosted-vespa:tenant-host:default. + APPLICATION_ID); + public static final UnboundDoubleFlag DEFAULT_TERM_WISE_LIMIT = defineDoubleFlag( "default-term-wise-limit", 1.0, List.of("baldersheim"), "2020-12-02", "2023-12-31", @@ -346,6 +356,13 @@ public class Flags { "Takes effect at redeployment", APPLICATION_ID); + public static final UnboundBooleanFlag VESPA_ATHENZ_PROVIDER = defineFeatureFlag( + "vespa-athenz-provider", false, + List.of("mortent"), "2023-02-22", "2023-05-01", + "Enable athenz provider in public systems", + "Takes effect on next config server container start", + ZONE_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/linguistics/src/main/java/com/yahoo/language/Linguistics.java b/linguistics/src/main/java/com/yahoo/language/Linguistics.java index 791bc5dabcf..6fa63e657bd 100644 --- a/linguistics/src/main/java/com/yahoo/language/Linguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/Linguistics.java @@ -47,7 +47,7 @@ public interface Linguistics { /** * Returns a thread-unsafe tokenizer. - * This is used at indexing time to produce a optionally stemmed and + * This is used at indexing time to produce an optionally stemmed and * transformed (accent normalized) stream of indexable tokens. */ Tokenizer getTokenizer(); diff --git a/metrics/src/tests/metricmanagertest.cpp b/metrics/src/tests/metricmanagertest.cpp index 52b17576c5c..9e6b0f40be3 100644 --- a/metrics/src/tests/metricmanagertest.cpp +++ b/metrics/src/tests/metricmanagertest.cpp @@ -29,7 +29,7 @@ struct MetricManagerTest : public ::testing::Test { // MetricManager that aren't accessible to "freestanding" fixtures. So we // get the test to do the necessary poking and prodding for us instead. void takeSnapshots(MetricManager& mm, time_t timeToProcess) { - mm.takeSnapshots(mm.getMetricLock(), system_time(vespalib::from_s(timeToProcess))); + mm.takeSnapshots(mm.getMetricLock(), system_time(vespalib::from_s<system_time::duration>(timeToProcess))); } }; @@ -364,7 +364,7 @@ class FakeTimer : public MetricManager::Timer { std::atomic<time_t> _time; public: FakeTimer(time_t startTime = 0) : _time(startTime) {} - time_point getTime() const override { return time_point(vespalib::from_s(load_relaxed(_time))); } + time_point getTime() const override { return time_point(vespalib::from_s<time_point::duration>(load_relaxed(_time))); } void set_time(time_t t) noexcept { store_relaxed(_time, t); } // Not safe for multiple writers, only expected to be called by test. void add_time(time_t t) noexcept { set_time(load_relaxed(_time) + t); } @@ -384,7 +384,7 @@ struct BriefValuePrinter : public MetricVisitor { } }; -bool waitForTimeProcessed(const MetricManager& mm, vespalib::duration processtime, uint32_t timeout = 120) +bool waitForTimeProcessed(const MetricManager& mm, time_point::duration processtime, uint32_t timeout = 120) { uint32_t lastchance = time(0) + timeout; while (time(0) < lastchance) { @@ -945,7 +945,7 @@ namespace { std::mutex& _output_mutex; FakeTimer& _timer; - MyUpdateHook(std::ostringstream& output, std::mutex& output_mutex, const char* name, vespalib::duration period, FakeTimer& timer) + MyUpdateHook(std::ostringstream& output, std::mutex& output_mutex, const char* name, vespalib::system_clock::duration period, FakeTimer& timer) : UpdateHook(name, period), _output(output), _output_mutex(output_mutex), diff --git a/metrics/src/tests/snapshottest.cpp b/metrics/src/tests/snapshottest.cpp index 5561825e2ba..580769bbadb 100644 --- a/metrics/src/tests/snapshottest.cpp +++ b/metrics/src/tests/snapshottest.cpp @@ -148,7 +148,7 @@ TestMetricSet::incValues() { struct FakeTimer : public MetricManager::Timer { uint32_t _timeInSecs; FakeTimer() : _timeInSecs(1) {} - time_point getTime() const override { return time_point(vespalib::from_s(_timeInSecs)); } + time_point getTime() const override { return time_point(vespalib::from_s<time_point::duration>(_timeInSecs)); } }; void ASSERT_VALUE(int32_t value, const MetricSnapshot & snapshot, const char *name) __attribute__((noinline)); @@ -166,7 +166,7 @@ void ASSERT_VALUE(int32_t value, const MetricSnapshot & snapshot, const char *na struct SnapshotTest : public ::testing::Test { void tick(MetricManager& mgr, time_t currentTime) { - mgr.tick(mgr.getMetricLock(), time_point(vespalib::from_s(currentTime))); + mgr.tick(mgr.getMetricLock(), time_point(vespalib::from_s<time_point::duration>(currentTime))); } }; diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp index 606b97bda73..2f6fe4c6ba6 100644 --- a/metrics/src/vespa/metrics/metricmanager.cpp +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -427,7 +427,7 @@ MetricManager::createSnapshotPeriods(const Config& config) } else { name << length << " seconds"; } - result.emplace_back(vespalib::from_s(length), name.str()); + result.emplace_back(vespalib::from_s<time_point::duration>(length), name.str()); } for (uint32_t i=1; i<result.size(); ++i) { if (result[i].first % result[i-1].first != vespalib::duration::zero()) { @@ -473,7 +473,7 @@ MetricManager::configure(const MetricLockGuard & , std::unique_ptr<Config> confi uint32_t nextCount = 1; if (i + 1 < snapshotPeriods.size()) { nextCount = snapshotPeriods[i + 1].first / snapshotPeriods[i].first; - if ((snapshotPeriods[i + 1].first % snapshotPeriods[i].first) != vespalib::duration::zero()) { + if ((snapshotPeriods[i + 1].first % snapshotPeriods[i].first) != time_point::duration::zero()) { throw IllegalStateException("Snapshot periods must be multiplum of each other",VESPA_STRLOC); } } @@ -570,11 +570,11 @@ MetricManager::visit(const MetricLockGuard & guard, const MetricSnapshot& snapsh visitor.doneVisiting(); } -std::vector<vespalib::duration> +std::vector<time_point::duration> MetricManager::getSnapshotPeriods(const MetricLockGuard& l) const { assertMetricLockLocked(l); - std::vector<vespalib::duration> result; + std::vector<time_point::duration> result; result.reserve(_snapshots.size()); for (const auto & snapshot : _snapshots) { result.emplace_back(snapshot->getPeriod()); diff --git a/metrics/src/vespa/metrics/metricmanager.h b/metrics/src/vespa/metrics/metricmanager.h index 99fad4f2ad3..6f40e7961f4 100644 --- a/metrics/src/vespa/metrics/metricmanager.h +++ b/metrics/src/vespa/metrics/metricmanager.h @@ -243,7 +243,7 @@ public: const MetricSnapshot& getMetricSnapshot( const MetricLockGuard&, vespalib::duration period, bool getInProgressSet) const; const MetricSnapshotSet& getMetricSnapshotSet(const MetricLockGuard&, vespalib::duration period) const; - std::vector<vespalib::duration> getSnapshotPeriods(const MetricLockGuard& l) const; + std::vector<time_point::duration> getSnapshotPeriods(const MetricLockGuard& l) const; // Public only for testing. The returned pointer is only valid while holding the lock. const ConsumerSpec * getConsumerSpec(const MetricLockGuard & guard, const Metric::String& consumer) const; @@ -290,7 +290,7 @@ private: void handleMetricsAltered(const MetricLockGuard & guard); - using SnapSpec = std::pair<vespalib::duration, std::string>; + using SnapSpec = std::pair<time_point::duration, std::string>; static std::vector<SnapSpec> createSnapshotPeriods( const MetricsmanagerConfig& config); void assertMetricLockLocked(const MetricLockGuard& g) const; }; diff --git a/metrics/src/vespa/metrics/metricsnapshot.cpp b/metrics/src/vespa/metrics/metricsnapshot.cpp index cd06fb731c2..6bcdcc60995 100644 --- a/metrics/src/vespa/metrics/metricsnapshot.cpp +++ b/metrics/src/vespa/metrics/metricsnapshot.cpp @@ -24,7 +24,7 @@ MetricSnapshot::MetricSnapshot(const Metric::String& name) { } -MetricSnapshot::MetricSnapshot(const Metric::String& name, vespalib::duration period, const MetricSet& source, bool copyUnset) +MetricSnapshot::MetricSnapshot(const Metric::String& name, system_time::duration period, const MetricSet& source, bool copyUnset) : _name(name), _period(period), _fromTime(system_time_epoch), @@ -73,7 +73,7 @@ MetricSnapshot::addMemoryUsage(MemoryConsumption& mc) const _snapshot->addMemoryUsage(mc); } -MetricSnapshotSet::MetricSnapshotSet(const Metric::String& name, vespalib::duration period, uint32_t count, +MetricSnapshotSet::MetricSnapshotSet(const Metric::String& name, system_time::duration period, uint32_t count, const MetricSet& source, bool snapshotUnsetMetrics) : _count(count), _builderCount(0), diff --git a/metrics/src/vespa/metrics/metricsnapshot.h b/metrics/src/vespa/metrics/metricsnapshot.h index 945f9dc7326..859ee4a4a97 100644 --- a/metrics/src/vespa/metrics/metricsnapshot.h +++ b/metrics/src/vespa/metrics/metricsnapshot.h @@ -23,7 +23,7 @@ class MetricSnapshot { Metric::String _name; // Period length of this snapshot - vespalib::duration _period; + system_time::duration _period; // Time this snapshot was last updated. system_time _fromTime; // If set to 0, use _fromTime + _period. @@ -37,7 +37,7 @@ public: /** Create a fresh empty top level snapshot. */ MetricSnapshot(const Metric::String& name); /** Create a snapshot of another metric source. */ - MetricSnapshot(const Metric::String& name, vespalib::duration period, + MetricSnapshot(const Metric::String& name, system_time::duration period, const MetricSet& source, bool copyUnset); ~MetricSnapshot(); @@ -54,7 +54,7 @@ public: void setToTime(system_time toTime) { _toTime = toTime; } const Metric::String& getName() const { return _name; } - vespalib::duration getPeriod() const { return _period; } + system_time::duration getPeriod() const { return _period; } system_time getFromTime() const { return _fromTime; } system_time getToTime() const { return _toTime; } const MetricSet& getMetrics() const { return *_snapshot; } @@ -78,11 +78,11 @@ class MetricSnapshotSet { std::unique_ptr<MetricSnapshot> _current; // The last full period std::unique_ptr<MetricSnapshot> _building; // The building period public: - MetricSnapshotSet(const Metric::String& name, vespalib::duration period, uint32_t count, + MetricSnapshotSet(const Metric::String& name, system_time::duration period, uint32_t count, const MetricSet& source, bool snapshotUnsetMetrics); const Metric::String& getName() const { return _current->getName(); } - vespalib::duration getPeriod() const { return _current->getPeriod(); } + system_time::duration getPeriod() const { return _current->getPeriod(); } system_time getFromTime() const { return _current->getFromTime(); } system_time getToTime() const { return _current->getToTime(); } system_time getNextWorkTime() const { return getToTime() + getPeriod(); } diff --git a/metrics/src/vespa/metrics/updatehook.h b/metrics/src/vespa/metrics/updatehook.h index 997bdf0b5a4..aced45b91c9 100644 --- a/metrics/src/vespa/metrics/updatehook.h +++ b/metrics/src/vespa/metrics/updatehook.h @@ -28,7 +28,7 @@ class MetricManager; class UpdateHook { public: using MetricLockGuard = metrics::MetricLockGuard; - UpdateHook(const char* name, vespalib::duration period) + UpdateHook(const char* name, time_point::duration period) : _name(name), _period(period), _nextCall() @@ -38,15 +38,15 @@ public: const char* getName() const { return _name; } void updateNextCall() { updateNextCall(_nextCall); } void updateNextCall(time_point now) { setNextCall(now + _period); } - bool is_periodic() const noexcept { return _period != vespalib::duration::zero(); } + bool is_periodic() const noexcept { return _period != time_point::duration::zero(); } bool expired(time_point now) { return _nextCall <= now; } bool has_valid_expiry() const noexcept { return _nextCall != time_point(); } - vespalib::duration getPeriod() const noexcept { return _period; } + time_point::duration getPeriod() const noexcept { return _period; } time_point getNextCall() const noexcept { return _nextCall; } void setNextCall(time_point now) { _nextCall = now; } private: const char* _name; - const vespalib::duration _period; + const time_point::duration _period; time_point _nextCall; }; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index fc49dcc744c..6bd7d98e207 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -41,6 +41,7 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; @@ -189,11 +190,9 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { Pkcs10Csr csr = csrGenerator.generateInstanceCsr( context.identity(), doc.providerUniqueId(), doc.ipAddresses(), doc.clusterType(), keyPair); - // Set up a hostname verified for zts if this is configured to use the config server (internal zts) apis - HostnameVerifier ztsHostNameVerifier = useInternalZts - ? new AthenzIdentityVerifier(Set.of(configserverIdentity)) - : null; - try (ZtsClient ztsClient = new DefaultZtsClient.Builder(ztsEndpoint).withIdentityProvider(hostIdentityProvider).withHostnameVerifier(ztsHostNameVerifier).build()) { + // Allow all zts hosts while removing SIS + HostnameVerifier ztsHostNameVerifier = (hostname, sslSession) -> true; + try (ZtsClient ztsClient = new DefaultZtsClient.Builder(ztsEndpoint(doc)).withIdentityProvider(hostIdentityProvider).withHostnameVerifier(ztsHostNameVerifier).build()) { InstanceIdentity instanceIdentity = ztsClient.registerInstance( configserverIdentity, @@ -206,6 +205,15 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { } } + /** + * Return zts url from identity document, fallback to ztsEndpoint + */ + private URI ztsEndpoint(SignedIdentityDocument doc) { + return Optional.ofNullable(doc.ztsUrl()) + .filter(s -> !s.isBlank()) + .map(URI::create) + .orElse(ztsEndpoint); + } private void refreshIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile, ContainerPath identityDocumentFile, SignedIdentityDocument doc) { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); @@ -217,11 +225,9 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { .build(); try { - // Set up a hostname verified for zts if this is configured to use the config server (internal zts) apis - HostnameVerifier ztsHostNameVerifier = useInternalZts - ? new AthenzIdentityVerifier(Set.of(configserverIdentity)) - : null; - try (ZtsClient ztsClient = new DefaultZtsClient.Builder(ztsEndpoint).withSslContext(containerIdentitySslContext).withHostnameVerifier(ztsHostNameVerifier).build()) { + // Allow all zts hosts while removing SIS + HostnameVerifier ztsHostNameVerifier = (hostname, sslSession) -> true; + try (ZtsClient ztsClient = new DefaultZtsClient.Builder(ztsEndpoint(doc)).withSslContext(containerIdentitySslContext).withHostnameVerifier(ztsHostNameVerifier).build()) { InstanceIdentity instanceIdentity = ztsClient.refreshInstance( configserverIdentity, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java index faa360bbcb1..ed4cab5137c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java @@ -46,12 +46,6 @@ public class ArchiveUriManager { archiveUris.get().accountArchiveUris().get(node.cloudAccount()) : archiveUris.get().tenantArchiveUris().get(app.tenant())) .map(uri -> { - // TODO (freva): Remove when all URIs dont have tenant name in them anymore - String tenantSuffix = "/" + app.tenant().value() + "/"; - if (uri.endsWith(tenantSuffix)) return uri.substring(0, uri.length() - tenantSuffix.length() + 1); - return uri; - }) - .map(uri -> { StringBuilder sb = new StringBuilder(100).append(uri) .append(app.tenant().value()).append('/') .append(app.application().value()).append('/') diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index f52c4cc85f7..4020166a132 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -62,7 +62,7 @@ public class Autoscaler { clusterNodes, nodeRepository.metricsDb(), nodeRepository.clock()); - if (clusterModel.isEmpty()) return Autoscaling.empty(clusterModel.description()); + if (clusterModel.isEmpty()) return Autoscaling.empty(); if (! limits.isEmpty() && cluster.minResources().equals(cluster.maxResources())) return Autoscaling.dontScale(Autoscaling.Status.unavailable, "Autoscaling is not enabled", clusterModel); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java index 4e1523834e4..281d9efe51a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java @@ -113,10 +113,6 @@ public class ClusterModel { public ClusterSpec clusterSpec() { return clusterSpec; } public Cluster cluster() { return cluster; } - public String description() { - return nodeTimeseries.description(); - } - public boolean isEmpty() { return nodeTimeseries().isEmpty(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java index ff9de2fb633..d694085729f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.applications.Cluster; -import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; import java.time.Duration; import java.util.List; @@ -24,8 +23,6 @@ public class ClusterNodesTimeseries { /** The measurements for all nodes in this snapshot */ private final List<NodeTimeseries> timeseries; - private final String description; - public ClusterNodesTimeseries(Duration period, Cluster cluster, NodeList clusterNodes, MetricsDb db) { this.clusterNodes = clusterNodes; @@ -34,7 +31,6 @@ public class ClusterNodesTimeseries { // If either this is the case, or there is a generation change, we ignore // the first warmupWindow metrics. var timeseries = db.getNodeTimeseries(period.plus(warmupDuration.multipliedBy(4)), clusterNodes); - var initialTimeseries = timeseries; if (cluster.lastScalingEvent().isPresent()) { long currentGeneration = cluster.lastScalingEvent().get().generation(); timeseries = keepGenerationAfterWarmup(timeseries, currentGeneration); @@ -42,20 +38,11 @@ public class ClusterNodesTimeseries { timeseries = keep(timeseries, snapshot -> snapshot.inService() && snapshot.stable()); timeseries = keep(timeseries, snapshot -> ! snapshot.at().isBefore(db.clock().instant().minus(period))); this.timeseries = timeseries; - if (isEmpty() && initialTimeseries.size() > 0) - description = initialTimeseries.get(0).description(cluster.lastScalingEvent().map(ScalingEvent::generation).orElse(-1L), clusterNodes); - else - description = ""; } private ClusterNodesTimeseries(NodeList clusterNodes, List<NodeTimeseries> timeseries) { this.clusterNodes = clusterNodes; this.timeseries = timeseries; - this.description = ""; - } - - public String description() { - return description; } public boolean isEmpty() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java index c6f9c6fdd36..8b7a2bafc40 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java @@ -4,6 +4,10 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.collections.ListMap; import com.yahoo.collections.Pair; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.metrics.ContainerMetrics; +import com.yahoo.metrics.HostedNodeAdminMetrics; +import com.yahoo.metrics.SearchNodeMetrics; +import com.yahoo.metrics.StorageMetrics; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; @@ -21,6 +25,7 @@ import java.util.Map; import java.util.Optional; import static com.yahoo.metrics.ContainerMetrics.APPLICATION_GENERATION; +import static com.yahoo.metrics.ContainerMetrics.IN_SERVICE; /** * A response containing metrics for a collection of nodes. @@ -115,7 +120,7 @@ public class MetricsResponse { cpu { // a node resource @Override - public List<String> metricResponseNames() { return List.of("cpu.util"); } + public List<String> metricResponseNames() { return List.of(HostedNodeAdminMetrics.CPU_UTIL.baseName()); } @Override double computeFinal(ListMap<String, Double> values) { @@ -127,15 +132,16 @@ public class MetricsResponse { @Override public List<String> metricResponseNames() { - return List.of("content.proton.resource_usage.memory.average", "mem.util"); + return List.of(HostedNodeAdminMetrics.MEM_UTIL.baseName(), + SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_MEMORY.average()); } @Override double computeFinal(ListMap<String, Double> values) { - var valueList = values.get("content.proton.resource_usage.memory.average"); // prefer over mem.util + var valueList = values.get(SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_MEMORY.average()); // prefer over mem.util if ( ! valueList.isEmpty()) return valueList.get(0); - valueList = values.get("mem.util"); + valueList = values.get(HostedNodeAdminMetrics.MEM_UTIL.baseName()); if ( ! valueList.isEmpty()) return valueList.get(0) / 100; // % to ratio return 0; @@ -146,15 +152,16 @@ public class MetricsResponse { @Override public List<String> metricResponseNames() { - return List.of("content.proton.resource_usage.disk.average", "disk.util"); + return List.of(HostedNodeAdminMetrics.DISK_UTIL.baseName(), + SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_DISK.average()); } @Override double computeFinal(ListMap<String, Double> values) { - var valueList = values.get("content.proton.resource_usage.disk.average"); // prefer over mem.util + var valueList = values.get(SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_DISK.average()); // prefer over mem.util if ( ! valueList.isEmpty()) return valueList.get(0); - valueList = values.get("disk.util"); + valueList = values.get(HostedNodeAdminMetrics.DISK_UTIL.baseName()); if ( ! valueList.isEmpty()) return valueList.get(0) / 100; // % to ratio return 0; @@ -165,7 +172,7 @@ public class MetricsResponse { @Override public List<String> metricResponseNames() { - return List.of(APPLICATION_GENERATION.baseName() /*, "content.proton.config.generation" */); + return List.of(APPLICATION_GENERATION.last(), SearchNodeMetrics.CONTENT_PROTON_CONFIG_GENERATION.last()); } @Override @@ -177,7 +184,7 @@ public class MetricsResponse { inService { @Override - public List<String> metricResponseNames() { return List.of("in_service"); } + public List<String> metricResponseNames() { return List.of(IN_SERVICE.last()); } @Override double computeFinal(ListMap<String, Double> values) { @@ -190,8 +197,8 @@ public class MetricsResponse { @Override public List<String> metricResponseNames() { - return List.of("queries.rate", - "content.proton.documentdb.matching.queries.rate"); + return List.of(ContainerMetrics.QUERIES.rate(), + SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_QUERIES.rate()); } }, @@ -199,10 +206,11 @@ public class MetricsResponse { @Override public List<String> metricResponseNames() { - return List.of("feed.http-requests.rate", - "vds.filestor.allthreads.put.count.rate", - "vds.filestor.allthreads.remove.count.rate", - "vds.filestor.allthreads.update.count.rate"); } + return List.of(ContainerMetrics.FEED_HTTP_REQUESTS.rate(), + StorageMetrics.VDS_FILESTOR_ALLTHREADS_PUT_COUNT.rate(), + StorageMetrics.VDS_FILESTOR_ALLTHREADS_REMOVE_COUNT.rate(), + StorageMetrics.VDS_FILESTOR_ALLTHREADS_UPDATE_COUNT.rate()); + } }; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java index 5aea6858f60..c1ab8489f40 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java @@ -102,7 +102,6 @@ public class CuratorDb { db.create(archiveUrisPath); db.create(loadBalancersPath); provisionIndexCounter.initialize(100); - CuratorOperations.delete(root.append("archiveUris").toString()); // TODO (freva): March 2023 } /** Adds a set of nodes. Rollbacks/fails transaction if any node is not in the expected state. */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index 0fbe912812e..d0eb95e2d72 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -186,7 +186,7 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { return new MessageResponse("Updated " + patcher.application()); } } - else if (path.matches("/nodes/v2/archive/account/{key}") || path.matches("/nodes/v2/archive/tenant/{key}") || path.matches("/nodes/v2/archive/{key}") /* TODO (freva): Remove March 2023 */) { + else if (path.matches("/nodes/v2/archive/account/{key}") || path.matches("/nodes/v2/archive/tenant/{key}")) { String uri = requiredField(toSlime(request), "uri", Inspector::asString); return setArchiveUri(path.get("key"), Optional.of(uri), !path.getPath().segments().get(3).equals("account")); } @@ -229,7 +229,7 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { private HttpResponse handleDELETE(HttpRequest request) { Path path = new Path(request.getUri()); if (path.matches("/nodes/v2/node/{hostname}")) return deleteNode(path.get("hostname")); - if (path.matches("/nodes/v2/archive/account/{key}") || path.matches("/nodes/v2/archive/tenant/{key}") || path.matches("/nodes/v2/archive/{key}") /* TODO (freva): Remove March 2023) */) + if (path.matches("/nodes/v2/archive/account/{key}") || path.matches("/nodes/v2/archive/tenant/{key}")) return setArchiveUri(path.get("key"), Optional.empty(), !path.getPath().segments().get(3).equals("account")); if (path.matches("/nodes/v2/upgrade/firmware")) return cancelFirmwareCheckResponse(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java index 44c1c976355..894c0be2f54 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java @@ -60,18 +60,6 @@ public class ArchiveUriManagerTest { assertEquals("scheme://tenant-bucket/dir/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, accountSystem)).get()); // URI for tenant because non-enclave acocunt } - @Test - public void handles_uri_with_tenant_name() { - ApplicationId app1 = ApplicationId.from("vespa", "music", "main"); - ArchiveUriManager archiveUriManager = new ProvisioningTester.Builder().build().nodeRepository().archiveUriManager(); - archiveUriManager.setArchiveUri(app1.tenant(), Optional.of("scheme://tenant-bucket/vespa")); - assertEquals("scheme://tenant-bucket/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, null)).get()); - - // Archive URI ends with the tenant name - archiveUriManager.setArchiveUri(app1.tenant(), Optional.of("scheme://tenant-vespa/")); - assertEquals("scheme://tenant-vespa/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, null)).get()); - } - private Node createNode(ApplicationId appId, CloudAccount account) { Node.Builder nodeBuilder = Node.create("id", "h432a.prod.us-south-1.vespa.domain.tld", new Flavor(NodeResources.unspecified()), Node.State.parked, NodeType.tenant); Optional.ofNullable(appId) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java index e83880404f4..24697d02681 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java @@ -76,6 +76,7 @@ public class MetricsV2MetricsFetcherTest { assertEquals(0.15, values.get(0).getSecond().load().memory(), delta); assertEquals(0.20, values.get(0).getSecond().load().disk(), delta); assertEquals(3, values.get(0).getSecond().generation(), delta); + assertFalse(values.get(0).getSecond().inService()); assertTrue(values.get(0).getSecond().stable()); } @@ -108,114 +109,119 @@ public class MetricsV2MetricsFetcherTest { } final String cannedResponseForApplication1 = - "{\n" + - " \"nodes\": [\n" + - " {\n" + - " \"hostname\": \"host-1.yahoo.com\",\n" + - " \"role\": \"role0\",\n" + - " \"node\": {\n" + - " \"timestamp\": 1234,\n" + - " \"metrics\": [\n" + - " {\n" + - " \"values\": {\n" + - " \"cpu.util\": 16.2,\n" + - " \"mem.util\": 23.1,\n" + - " \"disk.util\": 82\n" + - " },\n" + - " \"dimensions\": {\n" + - " \"state\": \"active\"\n" + - " }\n" + - " }\n" + - " ]\n" + - " }\n" + - " },\n" + - " {\n" + - " \"hostname\": \"host-2.yahoo.com\",\n" + - " \"role\": \"role1\",\n" + - " \"node\": {\n" + - " \"timestamp\": 1200,\n" + - " \"metrics\": [\n" + - " {\n" + - " \"values\": {\n" + - " \"mem.util\": 30,\n" + - " \"disk.util\": 40\n" + - " },\n" + - " \"dimensions\": {\n" + - " \"state\": \"active\"\n" + - " }\n" + - " }\n" + - " ]\n" + - " },\n" + - " \"services\": [\n" + - " {\n" + - " \"name\": \"searchnode\",\n" + - " \"timestamp\": 1234,\n" + - " \"status\": {\n" + - " \"code\": \"up\"\n" + - " },\n" + - " \"metrics\": [\n" + - " {\n" + - " \"values\": {\n" + - " \"content.proton.documentdb.matching.queries.rate\": 20.5\n" + - " },\n" + - " \"dimensions\": {\n" + - " \"documentType\": \"music\"\n" + - " }\n" + - " },\n" + - " {\n" + - " \"values\": {\n" + - " \"content.proton.resource_usage.memory.average\": 0.35,\n" + - " \"content.proton.resource_usage.disk.average\": 0.45\n" + - " },\n" + - " \"dimensions\": {\n" + - " }\n" + - " },\n" + - " {\n" + - " \"values\": {\n" + - " \"content.proton.documentdb.matching.queries.rate\": 13.5\n" + - " },\n" + - " \"dimensions\": {\n" + - " \"documentType\": \"books\"\n" + - " }\n" + - " },\n" + - " {\n" + - " \"values\": {\n" + - " \"queries.rate\": 11.0\n" + - " },\n" + - " \"dimensions\": {\n" + - " }\n" + - " }\n" + - " ]\n" + - " }\n" + - " ]\n" + - " }\n" + - " ]\n" + - "}\n"; + """ + { + "nodes": [ + { + "hostname": "host-1.yahoo.com", + "role": "role0", + "node": { + "timestamp": 1234, + "metrics": [ + { + "values": { + "cpu.util": 16.2, + "mem.util": 23.1, + "disk.util": 82 + }, + "dimensions": { + "state": "active" + } + } + ] + } + }, + { + "hostname": "host-2.yahoo.com", + "role": "role1", + "node": { + "timestamp": 1200, + "metrics": [ + { + "values": { + "mem.util": 30, + "disk.util": 40 + }, + "dimensions": { + "state": "active" + } + } + ] + }, + "services": [ + { + "name": "searchnode", + "timestamp": 1234, + "status": { + "code": "up" + }, + "metrics": [ + { + "values": { + "content.proton.documentdb.matching.queries.rate": 20.5 + }, + "dimensions": { + "documentType": "music" + } + }, + { + "values": { + "content.proton.resource_usage.memory.average": 0.35, + "content.proton.resource_usage.disk.average": 0.45 + }, + "dimensions": { + } + }, + { + "values": { + "content.proton.documentdb.matching.queries.rate": 13.5 + }, + "dimensions": { + "documentType": "books" + } + }, + { + "values": { + "queries.rate": 11.0 + }, + "dimensions": { + } + } + ] + } + ] + } + ] + } + """; final String cannedResponseForApplication2 = - "{\n" + - " \"nodes\": [\n" + - " {\n" + - " \"hostname\": \"host-3.yahoo.com\",\n" + - " \"role\": \"role0\",\n" + - " \"node\": {\n" + - " \"timestamp\": 1300,\n" + - " \"metrics\": [\n" + - " {\n" + - " \"values\": {\n" + - " \"cpu.util\": 10,\n" + - " \"mem.util\": 15,\n" + - " \"disk.util\": 20,\n" + - " \"application_generation\": 3\n" + - " },\n" + - " \"dimensions\": {\n" + - " \"state\": \"active\"\n" + - " }\n" + - " }\n" + - " ]\n" + - " }\n" + - " }\n" + - " ]\n" + - "}\n"; + """ + { + "nodes": [ + { + "hostname": "host-3.yahoo.com", + "role": "role0", + "node": { + "timestamp": 1300, + "metrics": [ + { + "values": { + "cpu.util": 10, + "mem.util": 15, + "disk.util": 20, + "application_generation.last": 3, + "in_service.last": 0 + }, + "dimensions": { + "state": "active" + } + } + ] + } + } + ] + } + """; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java index d379513a8f9..c7c6e770fe3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** + * @author bratseth */ public class NodeMetricsDbMaintainerTest { @@ -56,53 +57,56 @@ public class NodeMetricsDbMaintainerTest { private static class MockHttpClient implements MetricsV2MetricsFetcher.AsyncHttpClient { + // this value asserted on above final String cannedResponse = - "{\n" + - " \"nodes\": [\n" + - " {\n" + - " \"hostname\": \"host-1.yahoo.com\",\n" + - " \"role\": \"role0\",\n" + - " \"node\": {\n" + - " \"timestamp\": 1300,\n" + - " \"metrics\": [\n" + - " {\n" + - " \"values\": {\n" + - " \"cpu.util\": 14,\n" + // this value asserted on above - " \"mem_total.util\": 15,\n" + - " \"disk.util\": 20,\n" + - " \"application_generation\": 3,\n" + - " \"in_service\": 1\n" + - " },\n" + - " \"dimensions\": {\n" + - " \"state\": \"active\"\n" + - " }\n" + - " }\n" + - " ]\n" + - " }\n" + - " },\n" + - " {\n" + - " \"hostname\": \"host-2.yahoo.com\",\n" + - " \"role\": \"role0\",\n" + - " \"node\": {\n" + - " \"timestamp\": 1300,\n" + - " \"metrics\": [\n" + - " {\n" + - " \"values\": {\n" + - " \"cpu.util\": 1,\n" + - " \"mem_total.util\": 2,\n" + - " \"disk.util\": 3,\n" + - " \"application_generation\": 3,\n" + - " \"in_service\": 0\n" + - " },\n" + - " \"dimensions\": {\n" + - " \"state\": \"active\"\n" + - " }\n" + - " }\n" + - " ]\n" + - " }\n" + - " }\n" + - " ]\n" + - "}\n"; + """ + { + "nodes": [ + { + "hostname": "host-1.yahoo.com", + "role": "role0", + "node": { + "timestamp": 1300, + "metrics": [ + { + "values": { + "cpu.util": 14, + "mem_total.util": 15, + "disk.util": 20, + "application_generation.last": 3, + "in_service.last": 1 + }, + "dimensions": { + "state": "active" + } + } + ] + } + }, + { + "hostname": "host-2.yahoo.com", + "role": "role0", + "node": { + "timestamp": 1300, + "metrics": [ + { + "values": { + "cpu.util": 1, + "mem_total.util": 2, + "disk.util": 3, + "application_generation.last": 3, + "in_service.last": 0 + }, + "dimensions": { + "state": "active" + } + } + ] + } + } + ] + } + """; @Override public CompletableFuture<String> get(String url) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java index 03581146b9f..7fe2d77b647 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java @@ -39,7 +39,7 @@ public class ArchiveApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant/tenant3", Utf8.toBytes("{\"uri\": \"ftp://host/dir\"}"), Request.Method.PATCH), "{\"message\":\"Updated archive URI for tenant3\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant2", Utf8.toBytes("{\"uri\": \"s3://my-bucket/dir\"}"), Request.Method.PATCH), + assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant/tenant2", Utf8.toBytes("{\"uri\": \"s3://my-bucket/dir\"}"), Request.Method.PATCH), "{\"message\":\"Updated archive URI for tenant2\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/archive/account/777888999000", Utf8.toBytes("{\"uri\": \"s3://acc-bucket\"}"), Request.Method.PATCH), "{\"message\":\"Updated archive URI for 777888999000\"}"); @@ -27,7 +27,6 @@ <module>airlift-zstd</module> <module>application</module> <module>application-model</module> - <module>athenz-identity-provider-service</module> <module>bundle-plugin-test</module> <module>client</module> <module>cloud-tenant-base</module> diff --git a/screwdriver.yaml b/screwdriver.yaml index 9d423cacfe7..729847f7b5e 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -220,7 +220,7 @@ jobs: VESPA_REF=$(meta get vespa.ref) if [[ $VESPA_VERSION == null ]] || [[ $VESPA_REF == null ]]; then echo "Must have valid Vespa version and reference to continue (got VESPA_VERSION=$VESPA_VERSION, VESPA_REF=$VESPA_REF)." - exit 1 + return 1 fi meta set vespa.version $VESPA_VERSION - install-dependencies: | @@ -236,6 +236,95 @@ jobs: - update-sample-apps: | screwdriver/update-vespa-version-in-sample-apps.sh $VESPA_VERSION + publish-legacy-release: + image: docker.io/vespaengine/vespa-build-centos-stream8:latest + + annotations: + screwdriver.cd/cpu: 7 + screwdriver.cd/ram: 16 + screwdriver.cd/disk: HIGH + screwdriver.cd/timeout: 300 + screwdriver.cd/dockerEnabled: true + screwdriver.cd/dockerCpu: TURBO + screwdriver.cd/dockerRam: HIGH + screwdriver.cd/buildPeriodically: H 6 1 * * + + environment: + IMAGE_NAME: "vespaengine/vespa-generic-intel-x86_64" + + secrets: + - DOCKER_HUB_DEPLOY_KEY + + steps: + - get-vespa-version: | + set -x + VESPA_VERSION=$(meta get vespa.version --external sd@8683:publish-release) + if [[ $VESPA_VERSION == null ]] || [[ $VESPA_REF == null ]]; then + echo "Must have valid Vespa version to continue (got VESPA_VERSION=$VESPA_VERSION)." + return 1 + fi + - install-dependencies: | + dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + dnf install -y docker-ce docker-ce-cli containerd.io + docker system info + - checkout: | + mkdir -p workdir + cd workdir + export WORKDIR=$(pwd) + git clone -q https://github.com/vespa-engine/vespa + (cd vespa && git checkout v$VESPA_VERSION) + git clone -q https://github.com/vespa-engine/system-test + # Set correct version in pom.xml files + (cd vespa && screwdriver/replace-vespa-version-in-poms.sh $VESPA_VERSION $(pwd) ) + - build-rpms: | + cd $WORKDIR + make -C $WORKDIR/vespa -f .copr/Makefile srpm outdir=$WORKDIR + rpmbuild --rebuild \ + --define="_topdir $WORKDIR/vespa-rpmbuild" \ + --define "debug_package %{nil}" \ + --define "_debugsource_template %{nil}" \ + --define '_cmake_extra_opts "-DDEFAULT_VESPA_CPU_ARCH_FLAGS=-msse3 -mcx16 -mtune=intel"' \ + *.src.rpm + rm -f *.src.rpm + mv $WORKDIR/vespa-rpmbuild/RPMS/x86_64/*.rpm . + - build-container-image: | + cat <<EOF > Dockerfile + ARG VESPA_VERSION + FROM docker.io/vespaengine/vespa:\$VESPA_VERSION + USER root + RUN --mount=type=bind,target=/rpms/,source=. dnf reinstall -y /rpms/vespa*rpm && dnf clean all + USER vespa + EOF + docker build --progress plain --build-arg VESPA_VERSION=$VESPA_VERSION --tag docker.io/$IMAGE_NAME:$VESPA_VERSION \ + --tag docker.io/$IMAGE_NAME:latest --file Dockerfile . + - verify-container-image: | + # Trick to be able to use the documentation testing to verify the image built locally + docker tag docker.io/$IMAGE_NAME:$VESPA_VERSION vespaengine/vespa:latest + # Clone and setup doc tests + git clone -q --depth 1 https://github.com/vespa-engine/documentation + cd documentation + python3 -m pip install -qqq -r test/requirements.txt --user + echo -e "urls:\n - en/vespa-quick-start.html" > test/_quick-start.yaml + # Get the required vespa CLI + VESPA_CLI_VERSION=$(curl -fsSL https://api.github.com/repos/vespa-engine/vespa/releases/latest | grep -Po '"tag_name": "v\K.*?(?=")') && \ + curl -fsSL https://github.com/vespa-engine/vespa/releases/download/v${VESPA_CLI_VERSION}/vespa-cli_${VESPA_CLI_VERSION}_linux_amd64.tar.gz | tar -zxf - -C /opt && \ + ln -sf /opt/vespa-cli_${VESPA_CLI_VERSION}_linux_amd64/bin/vespa /usr/local/bin/ + # Run test + test/test.py -c test/_quick-start.yaml + - publish-test-image: | + if [[ -z $SD_PULL_REQUEST ]]; then + if curl -fsSL https://index.docker.io/v1/repositories/$IMAGE_NAME/tags/$VESPA_VERSION &> /dev/null; then + echo "Container image docker.io/$IMAGE_NAME:$VESPA_VERSION aldready exists." + else + OPT_STATE="$(set +o)" + set +x + docker login --username aressem --password "$DOCKER_HUB_DEPLOY_KEY" + eval "$OPT_STATE" + docker push docker.io/$IMAGE_NAME:$VESPA_VERSION + docker push docker.io/$IMAGE_NAME:latest + fi + fi + publish-cli-release: image: homebrew/brew:latest annotations: diff --git a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp index 7d9b1b3b06a..e412dabc7c1 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp @@ -83,6 +83,7 @@ getValue(const Context &context) const case BasicType::PREDICATE: case BasicType::TENSOR: case BasicType::REFERENCE: + case BasicType::RAW: throw IllegalArgumentException(make_string("Attribute '%s' of type '%s' can not be used for selection", v.getName().c_str(), BasicType(v.getBasicType()).asString())); case BasicType::MAX_TYPE: diff --git a/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp b/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp index c51771df265..2235f16ae94 100644 --- a/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp @@ -65,7 +65,7 @@ AttrVisitor::AttrVisitor(const search::IAttributeManager &amgr, CachedSelect::At AttrVisitor::~AttrVisitor() = default; bool isSingleValueThatWeHandle(BasicType type) { - return (type != BasicType::PREDICATE) && (type != BasicType::TENSOR) && (type != BasicType::REFERENCE); + return (type != BasicType::PREDICATE) && (type != BasicType::TENSOR) && (type != BasicType::REFERENCE) && (type != BasicType::RAW); } void diff --git a/searchlib/src/vespa/searchcommon/attribute/basictype.cpp b/searchlib/src/vespa/searchcommon/attribute/basictype.cpp index d0d90d1c9d5..41221457400 100644 --- a/searchlib/src/vespa/searchcommon/attribute/basictype.cpp +++ b/searchlib/src/vespa/searchcommon/attribute/basictype.cpp @@ -19,7 +19,8 @@ const BasicType::TypeInfo BasicType::_typeTable[BasicType::MAX_TYPE] = { { BasicType::DOUBLE, sizeof(double), "double" }, { BasicType::PREDICATE, 0, "predicate" }, { BasicType::TENSOR, 0, "tensor" }, - { BasicType::REFERENCE, 12, "reference" } + { BasicType::REFERENCE, 12, "reference" }, + { BasicType::RAW, 0, "raw" } }; BasicType::Type diff --git a/searchlib/src/vespa/searchcommon/attribute/basictype.h b/searchlib/src/vespa/searchcommon/attribute/basictype.h index bd7b4a2b4bc..46387dd2738 100644 --- a/searchlib/src/vespa/searchcommon/attribute/basictype.h +++ b/searchlib/src/vespa/searchcommon/attribute/basictype.h @@ -24,6 +24,7 @@ class BasicType PREDICATE = 11, TENSOR = 12, REFERENCE = 13, + RAW = 14, MAX_TYPE }; diff --git a/searchlib/src/vespa/searchcommon/attribute/iattributevector.h b/searchlib/src/vespa/searchcommon/attribute/iattributevector.h index 837aead18fd..34bf6f49cba 100644 --- a/searchlib/src/vespa/searchcommon/attribute/iattributevector.h +++ b/searchlib/src/vespa/searchcommon/attribute/iattributevector.h @@ -140,6 +140,13 @@ public: virtual const char * getString(DocId doc, char * buffer, size_t sz) const = 0; /** + * Return raw value. + * + * TODO: Consider accessing via new IRawAttribute interface class. + */ + virtual vespalib::ConstArrayRef<char> get_raw(DocId doc) const = 0; + + /** * Returns the first value stored for the given document as an enum value. * * @param docId document identifier @@ -370,6 +377,7 @@ public: virtual bool isPredicateType() const { return getBasicType() == BasicType::PREDICATE; } virtual bool isTensorType() const { return getBasicType() == BasicType::TENSOR; } virtual bool isReferenceType() const { return getBasicType() == BasicType::REFERENCE; } + virtual bool is_raw_type() const noexcept { return getBasicType() == BasicType::RAW; } /** * Returns whether this is a multi value attribute. diff --git a/searchlib/src/vespa/searchlib/attribute/floatbase.cpp b/searchlib/src/vespa/searchlib/attribute/floatbase.cpp index b9d7fb7c81b..31930aae061 100644 --- a/searchlib/src/vespa/searchlib/attribute/floatbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/floatbase.cpp @@ -95,6 +95,12 @@ FloatingPointAttribute::getString(DocId doc, char * s, size_t sz) const { return s; } +vespalib::ConstArrayRef<char> +FloatingPointAttribute::get_raw(DocId) const +{ + return {}; +} + vespalib::MemoryUsage FloatingPointAttribute::getChangeVectorMemoryUsage() const { diff --git a/searchlib/src/vespa/searchlib/attribute/floatbase.h b/searchlib/src/vespa/searchlib/attribute/floatbase.h index 288b66195e4..4eb1cdc8e02 100644 --- a/searchlib/src/vespa/searchlib/attribute/floatbase.h +++ b/searchlib/src/vespa/searchlib/attribute/floatbase.h @@ -38,6 +38,7 @@ protected: vespalib::MemoryUsage getChangeVectorMemoryUsage() const override; private: + vespalib::ConstArrayRef<char> get_raw(DocId) const override; uint32_t get(DocId doc, vespalib::string * v, uint32_t sz) const override; uint32_t get(DocId doc, const char ** v, uint32_t sz) const override; uint32_t get(DocId doc, WeightedString * v, uint32_t sz) const override; diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp index 489b2fb5e6e..f1125f34026 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp @@ -55,6 +55,12 @@ const char *ImportedAttributeVectorReadGuard::getString(DocId doc, char *buffer, return _target_attribute.getString(getTargetLid(doc), buffer, sz); } +vespalib::ConstArrayRef<char> +ImportedAttributeVectorReadGuard::get_raw(DocId doc) const +{ + return _target_attribute.get_raw(getTargetLid(doc)); +} + IAttributeVector::EnumHandle ImportedAttributeVectorReadGuard::getEnum(DocId doc) const { return _target_attribute.getEnum(getTargetLid(doc)); } diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h index fd9856a032c..f370696276e 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h @@ -43,6 +43,7 @@ public: largeint_t getInt(DocId doc) const override; double getFloat(DocId doc) const override; const char *getString(DocId doc, char *buffer, size_t sz) const override; + vespalib::ConstArrayRef<char> get_raw(DocId doc) const override; EnumHandle getEnum(DocId doc) const override; uint32_t get(DocId docId, largeint_t *buffer, uint32_t sz) const override; uint32_t get(DocId docId, double *buffer, uint32_t sz) const override; diff --git a/searchlib/src/vespa/searchlib/attribute/integerbase.cpp b/searchlib/src/vespa/searchlib/attribute/integerbase.cpp index b9d33f0aa9e..f77028f9464 100644 --- a/searchlib/src/vespa/searchlib/attribute/integerbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/integerbase.cpp @@ -68,6 +68,13 @@ IntegerAttribute::getString(DocId doc, char * s, size_t sz) const { } return s; } + +vespalib::ConstArrayRef<char> +IntegerAttribute::get_raw(DocId) const +{ + return {}; +} + uint32_t IntegerAttribute::get(DocId doc, vespalib::string * s, uint32_t sz) const { largeint_t * v = new largeint_t[sz]; diff --git a/searchlib/src/vespa/searchlib/attribute/integerbase.h b/searchlib/src/vespa/searchlib/attribute/integerbase.h index 7cb791e204e..f7de9ba9de4 100644 --- a/searchlib/src/vespa/searchlib/attribute/integerbase.h +++ b/searchlib/src/vespa/searchlib/attribute/integerbase.h @@ -37,6 +37,7 @@ protected: ChangeVector _changes; private: const char * getString(DocId doc, char * s, size_t sz) const override; + vespalib::ConstArrayRef<char> get_raw(DocId) const override; uint32_t get(DocId doc, vespalib::string * v, uint32_t sz) const override; uint32_t get(DocId doc, const char ** v, uint32_t sz) const override; uint32_t get(DocId doc, WeightedString * v, uint32_t sz) const override; diff --git a/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.cpp index d8b01afe094..3cbec4acb8b 100644 --- a/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.cpp @@ -48,6 +48,12 @@ NotImplementedAttribute::getString(DocId, char *, size_t) const { notImplemented(); } +vespalib::ConstArrayRef<char> +NotImplementedAttribute::get_raw(DocId) const +{ + notImplemented(); +} + uint32_t NotImplementedAttribute::get(DocId, largeint_t *, uint32_t) const { notImplemented(); diff --git a/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h b/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h index e824b0dd691..99fbdeab837 100644 --- a/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h +++ b/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h @@ -15,6 +15,7 @@ struct NotImplementedAttribute : AttributeVector { largeint_t getInt(DocId) const override; double getFloat(DocId) const override; const char * getString(DocId, char *, size_t) const override; + vespalib::ConstArrayRef<char> get_raw(DocId) const override; uint32_t get(DocId, largeint_t *, uint32_t) const override; uint32_t get(DocId, double *, uint32_t) const override; uint32_t get(DocId, vespalib::string *, uint32_t) const override; diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp index 3a9f88babfe..22a2eab1111 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp @@ -85,6 +85,12 @@ StringAttribute::getFloat(DocId doc) const { return vespalib::locale::c::strtod(get(doc), nullptr); } +vespalib::ConstArrayRef<char> +StringAttribute::get_raw(DocId) const +{ + return {}; +} + uint32_t StringAttribute::get(DocId doc, double * v, uint32_t sz) const { diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.h b/searchlib/src/vespa/searchlib/attribute/stringbase.h index e20a40d2df3..f40a89f76b4 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.h +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.h @@ -81,6 +81,7 @@ private: largeint_t getInt(DocId doc) const override { return strtoll(get(doc), nullptr, 0); } double getFloat(DocId doc) const override; + vespalib::ConstArrayRef<char> get_raw(DocId) const override; const char * getString(DocId doc, char * v, size_t sz) const override { (void) v; (void) sz; return get(doc); } long onSerializeForAscendingSort(DocId doc, void * serTo, long available, const common::BlobConverter * bc) const override; diff --git a/storage/src/vespa/storage/storageserver/statereporter.cpp b/storage/src/vespa/storage/storageserver/statereporter.cpp index d9e79d3b7b4..16de56fad22 100644 --- a/storage/src/vespa/storage/storageserver/statereporter.cpp +++ b/storage/src/vespa/storage/storageserver/statereporter.cpp @@ -73,7 +73,7 @@ StateReporter::getMetrics(const vespalib::string &consumer) if (periods.empty()) { return ""; // no configuration yet } - vespalib::duration interval = periods[0]; + auto interval = periods[0]; // To get unset metrics, we have to copy active metrics, clear them // and then assign the snapshot diff --git a/storage/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp b/storage/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp index 74d58244636..84b12d34e01 100644 --- a/storage/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp +++ b/storage/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp @@ -126,7 +126,7 @@ namespace { struct MetricHookWrapper : public metrics::UpdateHook { MetricUpdateHook& _hook; - MetricHookWrapper(vespalib::stringref name, MetricUpdateHook& hook, vespalib::duration period) + MetricHookWrapper(vespalib::stringref name, MetricUpdateHook& hook, vespalib::system_time::duration period) : metrics::UpdateHook(name.data(), period), // Expected to point to static name _hook(hook) { @@ -139,7 +139,7 @@ namespace { void ComponentRegisterImpl::registerUpdateHook(vespalib::stringref name, MetricUpdateHook& hook, - vespalib::duration period) + vespalib::system_time::duration period) { std::lock_guard lock(_componentLock); auto hookPtr = std::make_unique<MetricHookWrapper>(name, hook, period); diff --git a/storage/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h b/storage/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h index e569288ac64..43005575032 100644 --- a/storage/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h +++ b/storage/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h @@ -73,7 +73,7 @@ public: std::vector<const StatusReporter*> getStatusReporters() override; void registerMetric(metrics::Metric&) override; - void registerUpdateHook(vespalib::stringref name, MetricUpdateHook& hook, vespalib::duration period) override; + void registerUpdateHook(vespalib::stringref name, MetricUpdateHook& hook, vespalib::system_time::duration period) override; void registerShutdownListener(ShutdownListener&); }; diff --git a/storage/src/vespa/storageframework/generic/component/component.cpp b/storage/src/vespa/storageframework/generic/component/component.cpp index 0f08503852c..c69e59b8eba 100644 --- a/storage/src/vespa/storageframework/generic/component/component.cpp +++ b/storage/src/vespa/storageframework/generic/component/component.cpp @@ -52,7 +52,7 @@ Component::registerMetric(metrics::Metric& m) } void -Component::registerMetricUpdateHook(MetricUpdateHook& hook, vespalib::duration period) +Component::registerMetricUpdateHook(MetricUpdateHook& hook, vespalib::system_time::duration period) { assert(_metricUpdateHook.first == 0); _metricUpdateHook = std::make_pair(&hook, period); diff --git a/storage/src/vespa/storageframework/generic/component/component.h b/storage/src/vespa/storageframework/generic/component/component.h index 47469cce05d..372559e133d 100644 --- a/storage/src/vespa/storageframework/generic/component/component.h +++ b/storage/src/vespa/storageframework/generic/component/component.h @@ -86,7 +86,7 @@ class Component : private ManagedComponent metrics::Metric* _metric; ThreadPool* _threadPool; MetricRegistrator* _metricReg; - std::pair<MetricUpdateHook*, vespalib::duration> _metricUpdateHook; + std::pair<MetricUpdateHook*, vespalib::system_time::duration> _metricUpdateHook; const Clock* _clock; // ManagedComponent implementation @@ -124,7 +124,7 @@ public: * update hook will only be called if there actually is a metric mananger * component registered in the application. */ - void registerMetricUpdateHook(MetricUpdateHook&, vespalib::duration period); + void registerMetricUpdateHook(MetricUpdateHook&, vespalib::system_time::duration period); /** Get the name of the component. Must be a unique name. */ [[nodiscard]] const vespalib::string& getName() const override { return _name; } diff --git a/storage/src/vespa/storageframework/generic/metric/metricregistrator.h b/storage/src/vespa/storageframework/generic/metric/metricregistrator.h index 6daca1213a8..bea43fcfb6b 100644 --- a/storage/src/vespa/storageframework/generic/metric/metricregistrator.h +++ b/storage/src/vespa/storageframework/generic/metric/metricregistrator.h @@ -24,7 +24,7 @@ struct MetricRegistrator { virtual ~MetricRegistrator() = default; virtual void registerMetric(metrics::Metric&) = 0; - virtual void registerUpdateHook(vespalib::stringref name, MetricUpdateHook& hook, vespalib::duration period) = 0; + virtual void registerUpdateHook(vespalib::stringref name, MetricUpdateHook& hook, vespalib::system_time::duration period) = 0; }; } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java index 9b7b666e353..2d77d2ceda1 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java @@ -4,8 +4,10 @@ package com.yahoo.vespa.athenz.identityprovider.api; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; +import com.yahoo.vespa.athenz.utils.AthenzIdentities; import java.io.IOException; import java.io.InputStream; @@ -58,6 +60,8 @@ public class EntityBindingsMapper { entity.ipAddresses(), IdentityType.fromId(entity.identityType()), Optional.ofNullable(entity.clusterType()).map(ClusterType::from).orElse(null), + entity.ztsUrl(), + Optional.ofNullable(entity.serviceIdentity()).map(AthenzIdentities::from).orElse(null), entity.unknownAttributes()); } @@ -74,6 +78,8 @@ public class EntityBindingsMapper { model.ipAddresses(), model.identityType().id(), Optional.ofNullable(model.clusterType()).map(ClusterType::toConfigValue).orElse(null), + model.ztsUrl(), + Optional.ofNullable(model.serviceIdentity()).map(AthenzIdentity::getFullName).orElse(null), model.unknownAttributes()); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java index 49a39d25e87..de78d81cd1b 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java @@ -1,8 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.identityprovider.api; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; +import java.net.URL; import java.time.Instant; import java.util.HashMap; import java.util.Map; @@ -17,7 +19,8 @@ import java.util.Set; public record SignedIdentityDocument(String signature, int signingKeyVersion, VespaUniqueInstanceId providerUniqueId, AthenzService providerService, int documentVersion, String configServerHostname, String instanceHostname, Instant createdAt, Set<String> ipAddresses, - IdentityType identityType, ClusterType clusterType, Map<String, Object> unknownAttributes) { + IdentityType identityType, ClusterType clusterType, String ztsUrl, + AthenzIdentity serviceIdentity, Map<String, Object> unknownAttributes) { public SignedIdentityDocument { ipAddresses = Set.copyOf(ipAddresses); @@ -33,13 +36,19 @@ public record SignedIdentityDocument(String signature, int signingKeyVersion, Ve public SignedIdentityDocument(String signature, int signingKeyVersion, VespaUniqueInstanceId providerUniqueId, AthenzService providerService, int documentVersion, String configServerHostname, String instanceHostname, Instant createdAt, Set<String> ipAddresses, - IdentityType identityType, ClusterType clusterType) { + IdentityType identityType, ClusterType clusterType, String ztsUrl, AthenzIdentity serviceIdentity) { this(signature, signingKeyVersion, providerUniqueId, providerService, documentVersion, configServerHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, Map.of()); + instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity, Map.of()); } - public static final int DEFAULT_DOCUMENT_VERSION = 2; + public static final int DEFAULT_DOCUMENT_VERSION = 3; public boolean outdated() { return documentVersion < DEFAULT_DOCUMENT_VERSION; } + public SignedIdentityDocument withServiceIdentity(AthenzIdentity identity) { + return new SignedIdentityDocument(signature, signingKeyVersion, providerUniqueId, providerService, documentVersion, configServerHostname, instanceHostname, createdAt, + ipAddresses, identityType, clusterType, ztsUrl, identity); + } + + } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java index c37dd2f9147..fc0dff3b97b 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.athenz.identityprovider.api.bindings; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import java.time.Instant; @@ -14,10 +15,11 @@ import java.util.Set; /** * @author bjorncs */ +@JsonInclude(JsonInclude.Include.NON_NULL) public record SignedIdentityDocumentEntity( String signature, int signingKeyVersion, String providerUniqueId, String providerService, int documentVersion, String configServerHostname, String instanceHostname, Instant createdAt, Set<String> ipAddresses, - String identityType, String clusterType, Map<String, Object> unknownAttributes) { + String identityType, String clusterType, String ztsUrl, String serviceIdentity, Map<String, Object> unknownAttributes) { @JsonCreator public SignedIdentityDocumentEntity(@JsonProperty("signature") String signature, @@ -30,9 +32,11 @@ public record SignedIdentityDocumentEntity( @JsonProperty("created-at") Instant createdAt, @JsonProperty("ip-addresses") Set<String> ipAddresses, @JsonProperty("identity-type") String identityType, - @JsonProperty("cluster-type") String clusterType) { + @JsonProperty("cluster-type") String clusterType, + @JsonProperty("zts-url") String ztsUrl, + @JsonProperty("service-identity") String serviceIdentity) { this(signature, signingKeyVersion, providerUniqueId, providerService, documentVersion, configServerHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, new HashMap<>()); + instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity, new HashMap<>()); } @JsonProperty("signature") @Override public String signature() { return signature; } @@ -46,6 +50,8 @@ public record SignedIdentityDocumentEntity( @JsonProperty("ip-addresses") @Override public Set<String> ipAddresses() { return ipAddresses; } @JsonProperty("identity-type") @Override public String identityType() { return identityType; } @JsonProperty("cluster-type") @Override public String clusterType() { return clusterType; } + @JsonProperty("zts-url") @Override public String ztsUrl() { return ztsUrl; } + @JsonProperty("service-identity") @Override public String serviceIdentity() { return serviceIdentity; } @JsonAnyGetter @Override public Map<String, Object> unknownAttributes() { return unknownAttributes; } @JsonAnySetter public void set(String name, Object value) { unknownAttributes.put(name, value); } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java index 14d06fe83f2..019f73fc6bf 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.athenz.identityprovider.client; import com.yahoo.security.SignatureUtils; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; @@ -18,6 +19,7 @@ import java.util.Base64; import java.util.Set; import java.util.TreeSet; +import static com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION; import static java.nio.charset.StandardCharsets.UTF_8; /** @@ -35,13 +37,15 @@ public class IdentityDocumentSigner { Instant createdAt, Set<String> ipAddresses, IdentityType identityType, - PrivateKey privateKey) { + PrivateKey privateKey, + AthenzIdentity serviceIdentity) { try { Signature signer = SignatureUtils.createSigner(privateKey); signer.initSign(privateKey); writeToSigner( signer, providerUniqueId, providerService, configServerHostname, instanceHostname, createdAt, ipAddresses, identityType); + writeToSigner(signer, serviceIdentity); byte[] signature = signer.sign(); return Base64.getEncoder().encodeToString(signature); } catch (GeneralSecurityException e) { @@ -56,6 +60,9 @@ public class IdentityDocumentSigner { writeToSigner( signer, doc.providerUniqueId(), doc.providerService(), doc.configServerHostname(), doc.instanceHostname(), doc.createdAt(), doc.ipAddresses(), doc.identityType()); + if (doc.documentVersion() >= DEFAULT_DOCUMENT_VERSION) { + writeToSigner(signer, doc.serviceIdentity()); + } return signer.verify(Base64.getDecoder().decode(doc.signature())); } catch (GeneralSecurityException e) { throw new RuntimeException(e); @@ -82,4 +89,8 @@ public class IdentityDocumentSigner { } signer.update(identityType.id().getBytes(UTF_8)); } + + private static void writeToSigner(Signature signer, AthenzIdentity serviceIdentity) throws SignatureException{ + signer.update(serviceIdentity.getFullName().getBytes(UTF_8)); + } } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java index f8c119190a6..2a68f6fd231 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java @@ -30,6 +30,7 @@ class EntityBindingsMapperTest { "ip-addresses": [], "identity-type": "node", "cluster-type": "admin", + "zts-url": "https://zts.url/", "unknown-string": "string-value", "unknown-object": { "member-in-unknown-object": 123 } } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java index 0b8ff4277f1..ff85cb79f02 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java @@ -3,11 +3,13 @@ package com.yahoo.vespa.athenz.identityprovider.client; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.ClusterType; import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; +import com.yahoo.vespa.athenz.utils.AthenzIdentities; import org.junit.jupiter.api.Test; import java.security.KeyPair; @@ -36,37 +38,54 @@ public class IdentityDocumentSignerTest { private static final Instant createdAt = Instant.EPOCH; private static final HashSet<String> ipAddresses = new HashSet<>(Arrays.asList("1.2.3.4", "::1")); private static final ClusterType clusterType = ClusterType.CONTAINER; + private static final String ztsUrl = "https://foo"; + private static final AthenzIdentity serviceIdentity = new AthenzService("vespa", "node"); @Test void generates_and_validates_signature() { IdentityDocumentSigner signer = new IdentityDocumentSigner(); String signature = signer.generateSignature(id, providerService, configserverHostname, instanceHostname, createdAt, - ipAddresses, identityType, keyPair.getPrivate()); + ipAddresses, identityType, keyPair.getPrivate(), serviceIdentity); SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument( signature, KEY_VERSION, id, providerService, DEFAULT_DOCUMENT_VERSION, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType); + instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity); assertTrue(signer.hasValidSignature(signedIdentityDocument, keyPair.getPublic())); } @Test - void ignores_cluster_type() { + void ignores_cluster_type_and_zts_url() { IdentityDocumentSigner signer = new IdentityDocumentSigner(); String signature = signer.generateSignature(id, providerService, configserverHostname, instanceHostname, createdAt, - ipAddresses, identityType, keyPair.getPrivate()); + ipAddresses, identityType, keyPair.getPrivate(), serviceIdentity); - var docWithoutClusterType = new SignedIdentityDocument( + var docWithoutIgnoredFields = new SignedIdentityDocument( signature, KEY_VERSION, id, providerService, DEFAULT_DOCUMENT_VERSION, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, null); - var docWithClusterType = new SignedIdentityDocument( + instanceHostname, createdAt, ipAddresses, identityType, null, null, serviceIdentity); + var docWithIgnoredFields = new SignedIdentityDocument( signature, KEY_VERSION, id, providerService, DEFAULT_DOCUMENT_VERSION, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType); + instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity); + + assertTrue(signer.hasValidSignature(docWithoutIgnoredFields, keyPair.getPublic())); + assertEquals(docWithIgnoredFields.signature(), docWithoutIgnoredFields.signature()); + } + + @Test + void validates_signature_for_new_and_old_versions() { + IdentityDocumentSigner signer = new IdentityDocumentSigner(); + String signature = + signer.generateSignature(id, providerService, configserverHostname, instanceHostname, createdAt, + ipAddresses, identityType, keyPair.getPrivate(), serviceIdentity); + + SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument( + signature, KEY_VERSION, id, providerService, DEFAULT_DOCUMENT_VERSION, configserverHostname, + instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity); + + assertTrue(signer.hasValidSignature(signedIdentityDocument, keyPair.getPublic())); - assertTrue(signer.hasValidSignature(docWithoutClusterType, keyPair.getPublic())); - assertEquals(docWithClusterType.signature(), docWithoutClusterType.signature()); } }
\ No newline at end of file diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 66fd4b0895e..02f75c19907 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -28,7 +28,6 @@ com.google.code.findbugs:jsr305:3.0.2 com.google.errorprone:error_prone_annotations:2.18.0 com.google.guava:failureaccess:1.0.1 com.google.guava:guava:27.1-jre -com.google.inject:guice:4.2.3 com.google.inject:guice:4.2.3:no_aop com.google.j2objc:j2objc-annotations:1.1 com.google.protobuf:protobuf-java:3.21.7 @@ -221,6 +220,7 @@ xml-apis:xml-apis:1.4.01 com.github.luben:zstd-jni:1.5.2-1 com.github.tomakehurst:wiremock-jre8-standalone:2.35.0 com.google.guava:guava-testlib:27.1-jre +com.google.inject:guice:4.2.3 com.google.jimfs:jimfs:1.2 junit:junit:4.13.2 net.bytebuddy:byte-buddy:1.11.19 diff --git a/vespabase/src/vespa-configserver.service.in b/vespabase/src/vespa-configserver.service.in index f160c2ea8ad..7113ea501c1 100644 --- a/vespabase/src/vespa-configserver.service.in +++ b/vespabase/src/vespa-configserver.service.in @@ -5,6 +5,7 @@ After=network.target [Service] Type=forking +User=vespa PIDFile=@CMAKE_INSTALL_PREFIX@/var/run/configserver.pid ExecStart=@CMAKE_INSTALL_PREFIX@/bin/vespa-start-configserver ExecStop=@CMAKE_INSTALL_PREFIX@/bin/vespa-stop-configserver diff --git a/vespabase/src/vespa.service.in b/vespabase/src/vespa.service.in index e10724ef736..707d67ff98f 100644 --- a/vespabase/src/vespa.service.in +++ b/vespabase/src/vespa.service.in @@ -5,6 +5,7 @@ After=network.target [Service] Type=forking +User=vespa PIDFile=@CMAKE_INSTALL_PREFIX@/var/run/sentinel.pid ExecStart=@CMAKE_INSTALL_PREFIX@/bin/vespa-start-services ExecStop=@CMAKE_INSTALL_PREFIX@/bin/vespa-stop-services diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java index 288df7e470c..c2dea5e563b 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java @@ -55,6 +55,7 @@ public class StdOutVisitorHandler extends VdsVisitHandler { OutputFormat outputFormat = OutputFormat.JSON; boolean tensorShortForm = false; // TODO Vespa 9: change default to true boolean tensorDirectValues = false; // TODO Vespa 9: change default to true + boolean nullRender = false; boolean usesJson() { return outputFormat == OutputFormat.JSON || outputFormat == OutputFormat.JSONL; @@ -157,6 +158,9 @@ public class StdOutVisitorHandler extends VdsVisitHandler { @Override public void onDocument(Document doc, long timestamp) { try { + if (params.nullRender) { + return; + } if (lastLineIsProgress) { System.err.print('\r'); } @@ -187,6 +191,9 @@ public class StdOutVisitorHandler extends VdsVisitHandler { @Override public void onRemove(DocumentId docId) { try { + if (params.nullRender) { + return; + } if (lastLineIsProgress) { System.err.print('\r'); } @@ -263,7 +270,7 @@ public class StdOutVisitorHandler extends VdsVisitHandler { @Override public synchronized void onDone() { - if ((params.outputFormat == OutputFormat.JSON) && !params.printIds) { + if ((params.outputFormat == OutputFormat.JSON) && !params.printIds && !params.nullRender) { if (first) { out.print('['); } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java index f2ddd4ed8ea..8b919f7e9ea 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java @@ -381,6 +381,13 @@ public class VdsVisit { .type(Number.class) .build()); + options.addOption(Option.builder() + .longOpt("nullrender") + .desc("Process documents, but do not render any output. Overrides all other output options. " + + "Used to benchmark whether document rendering is the bottleneck when processing documents.") + .hasArg(false) + .build()); + return options; } @@ -399,6 +406,7 @@ public class VdsVisit { private boolean jsonLinesOutput = false; private boolean tensorShortForm = false; // TODO Vespa 9: change default to true private boolean tensorDirectValues = false; // TODO Vespa 9: change default to true + private boolean nullRender = false; private int slices = 1; private int sliceId = 0; @@ -508,6 +516,14 @@ public class VdsVisit { this.tensorDirectValues = tensorDirectValues; } + public boolean nullRender() { + return nullRender; + } + + public void setNullRender(boolean nullRender) { + this.nullRender = nullRender; + } + public int slices() { return slices; } @@ -660,6 +676,9 @@ public class VdsVisit { if (line.hasOption("tensorvalues")) { allParams.setTensorDirectValues(true); } + if (line.hasOption("nullrender")) { + allParams.setNullRender(true); + } if (line.hasOption("slices") != line.hasOption("sliceid")) { throw new IllegalArgumentException("Both --slices and --sliceid must be specified when visiting with slicing"); } @@ -848,6 +867,7 @@ public class VdsVisit { handlerParams.outputFormat = params.stdOutHandlerOutputFormat(); handlerParams.tensorShortForm = params.tensorShortForm(); handlerParams.tensorDirectValues = params.tensorDirectValues(); + handlerParams.nullRender = params.nullRender(); handler = new StdOutVisitorHandler(handlerParams); if (visitorParameters.getResumeFileName() != null) { diff --git a/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java b/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java index aa708b1fde9..1ebbc8ac6ad 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java @@ -156,4 +156,27 @@ public class StdOutVisitorHandlerTest { } } + @Test + void nothing_is_rendered_if_null_render_option_is_specified() { + var docType = new DocumentType("foo"); + docType.addField("bar", DataType.STRING); + + var params = createHandlerParams(true, true, true); + params.nullRender = true; + + var out = new ByteArrayOutputStream(); + var visitorHandler = new StdOutVisitorHandler(params, new PrintStream(out, true)); + var dataHandler = visitorHandler.getDataHandler(); + var controlSession = mock(VisitorControlSession.class); + dataHandler.setSession(controlSession); + + dataHandler.onMessage(createPutWithDocAndValue(docType, "id:baz:foo::1", "fluffy\nbunnies"), mock(AckToken.class)); + dataHandler.onMessage(createRemoveForDoc("id:baz:foo::2"), mock(AckToken.class)); + dataHandler.onMessage(createPutWithDocAndValue(docType, "id:baz:foo::3", "\r\ncool fox\r\n"), mock(AckToken.class)); + dataHandler.onDone(); + + String output = out.toString().trim(); + assertEquals("", output); + } + } diff --git a/vespalib/src/vespa/vespalib/util/time.h b/vespalib/src/vespa/vespalib/util/time.h index 27f359071ae..2cb53df8ae2 100644 --- a/vespalib/src/vespa/vespalib/util/time.h +++ b/vespalib/src/vespa/vespalib/util/time.h @@ -43,8 +43,9 @@ constexpr double to_s(duration d) { system_time to_utc(steady_time ts); -constexpr duration from_s(double seconds) { - return std::chrono::duration_cast<duration>(std::chrono::duration<double>(seconds)); +template <typename duration_type = duration> +constexpr duration_type from_s(double seconds) { + return std::chrono::duration_cast<duration_type>(std::chrono::duration<double>(seconds)); } constexpr int64_t count_s(duration d) { |