diff options
Diffstat (limited to 'configserver')
9 files changed, 30 insertions, 88 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 8eb9c66150f..596f9c0ce98 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -164,6 +164,7 @@ public class ModelContextImpl implements ModelContext { private final double feedConcurrency; private final boolean reconfigurableZookeeperServer; private final boolean enableJdiscConnectionLog; + private final boolean enableZstdCompressionAccessLog; public FeatureFlags(FlagSource source, ApplicationId appId) { this.enableAutomaticReindexing = flagValue(source, appId, Flags.ENABLE_AUTOMATIC_REINDEXING); @@ -183,6 +184,7 @@ public class ModelContextImpl implements ModelContext { this.feedConcurrency = flagValue(source, appId, Flags.FEED_CONCURRENCY); this.reconfigurableZookeeperServer = flagValue(source, appId, Flags.RECONFIGURABLE_ZOOKEEPER_SERVER_FOR_CLUSTER_CONTROLLER); this.enableJdiscConnectionLog = flagValue(source, appId, Flags.ENABLE_JDISC_CONNECTION_LOG); + this.enableZstdCompressionAccessLog = flagValue(source, appId, Flags.ENABLE_ZSTD_COMPRESSION_ACCESS_LOG); } @Override public boolean enableAutomaticReindexing() { return enableAutomaticReindexing; } @@ -202,6 +204,7 @@ public class ModelContextImpl implements ModelContext { @Override public double feedConcurrency() { return feedConcurrency; } @Override public boolean reconfigurableZookeeperServer() { return reconfigurableZookeeperServer; } @Override public boolean enableJdiscConnectionLog() { return enableJdiscConnectionLog; } + @Override public boolean enableZstdCompressionAccessLog() { return enableZstdCompressionAccessLog; } private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 75ab09d241b..b80037eef56 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -23,6 +23,7 @@ import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.provision.ProvisionerAdapter; import com.yahoo.vespa.config.server.provision.StaticProvisioner; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collections; @@ -79,6 +80,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { ApplicationPackage applicationPackage, SettableOptional<AllocatedHosts> allocatedHosts, Instant now) { + Instant start = Instant.now(); log.log(Level.FINE, "Will build models for " + applicationId); Set<Version> versions = modelFactoryRegistry.allVersions(); @@ -132,10 +134,11 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { } } log.log(Level.FINE, "Done building models for " + applicationId + ". Built models for versions " + - allApplicationModels.stream() - .map(result -> result.getModel().version()) - .map(Version::toFullString) - .collect(Collectors.toSet())) ; + allApplicationModels.stream() + .map(result -> result.getModel().version()) + .map(Version::toFullString) + .collect(Collectors.toSet()) + + " in " + Duration.between(start, Instant.now())); return allApplicationModels; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index 139081fde00..78071cbf89e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -37,7 +37,6 @@ public final class PrepareParams { static final String VERBOSE_PARAM_NAME = "verbose"; static final String VESPA_VERSION_PARAM_NAME = "vespaVersion"; static final String CONTAINER_ENDPOINTS_PARAM_NAME = "containerEndpoints"; - static final String TLS_SECRETS_KEY_NAME_PARAM_NAME = "tlsSecretsKeyName"; static final String ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME = "endpointCertificateMetadata"; static final String DOCKER_IMAGE_REPOSITORY = "dockerImageRepository"; static final String ATHENZ_DOMAIN = "athenzDomain"; @@ -55,7 +54,6 @@ public final class PrepareParams { private final boolean force; private final Optional<Version> vespaVersion; private final List<ContainerEndpoint> containerEndpoints; - private final Optional<String> tlsSecretsKeyName; private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; private final Optional<DockerImage> dockerImageRepository; private final Optional<AthenzDomain> athenzDomain; @@ -64,7 +62,7 @@ public final class PrepareParams { private PrepareParams(ApplicationId applicationId, TimeoutBudget timeoutBudget, boolean ignoreValidationErrors, boolean dryRun, boolean verbose, boolean isBootstrap, Optional<Version> vespaVersion, - List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName, + List<ContainerEndpoint> containerEndpoints, Optional<EndpointCertificateMetadata> endpointCertificateMetadata, Optional<DockerImage> dockerImageRepository, Optional<AthenzDomain> athenzDomain, Optional<ApplicationRoles> applicationRoles, Optional<Quota> quota, boolean force) { @@ -76,7 +74,6 @@ public final class PrepareParams { this.isBootstrap = isBootstrap; this.vespaVersion = vespaVersion; this.containerEndpoints = containerEndpoints; - this.tlsSecretsKeyName = tlsSecretsKeyName; this.endpointCertificateMetadata = endpointCertificateMetadata; this.dockerImageRepository = dockerImageRepository; this.athenzDomain = athenzDomain; @@ -96,7 +93,6 @@ public final class PrepareParams { private TimeoutBudget timeoutBudget = new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(60)); private Optional<Version> vespaVersion = Optional.empty(); private List<ContainerEndpoint> containerEndpoints = null; - private Optional<String> tlsSecretsKeyName = Optional.empty(); private Optional<EndpointCertificateMetadata> endpointCertificateMetadata = Optional.empty(); private Optional<DockerImage> dockerImageRepository = Optional.empty(); private Optional<AthenzDomain> athenzDomain = Optional.empty(); @@ -156,12 +152,6 @@ public final class PrepareParams { return this; } - public Builder tlsSecretsKeyName(String tlsSecretsKeyName) { - this.tlsSecretsKeyName = Optional.ofNullable(tlsSecretsKeyName) - .filter(s -> ! s.isEmpty()); - return this; - } - public Builder endpointCertificateMetadata(String serialized) { this.endpointCertificateMetadata = (serialized == null) ? Optional.empty() @@ -210,7 +200,7 @@ public final class PrepareParams { public PrepareParams build() { return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun, - verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName, + verbose, isBootstrap, vespaVersion, containerEndpoints, endpointCertificateMetadata, dockerImageRepository, athenzDomain, applicationRoles, quota, force); } @@ -224,7 +214,6 @@ public final class PrepareParams { .applicationId(createApplicationId(request, tenant)) .vespaVersion(request.getProperty(VESPA_VERSION_PARAM_NAME)) .containerEndpoints(request.getProperty(CONTAINER_ENDPOINTS_PARAM_NAME)) - .tlsSecretsKeyName(request.getProperty(TLS_SECRETS_KEY_NAME_PARAM_NAME)) .endpointCertificateMetadata(request.getProperty(ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME)) .dockerImageRepository(request.getProperty(DOCKER_IMAGE_REPOSITORY)) .athenzDomain(request.getProperty(ATHENZ_DOMAIN)) @@ -284,10 +273,6 @@ public final class PrepareParams { return timeoutBudget; } - public Optional<String> tlsSecretsKeyName() { - return tlsSecretsKeyName; - } - public Optional<EndpointCertificateMetadata> endpointCertificateMetadata() { return endpointCertificateMetadata; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index c244274c49f..b29259e22d4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -181,8 +181,7 @@ public class SessionPreparer { this.containerEndpointsCache = new ContainerEndpointsCache(tenantPath, curator); this.endpointCertificateMetadataStore = new EndpointCertificateMetadataStore(curator, tenantPath); EndpointCertificateRetriever endpointCertificateRetriever = new EndpointCertificateRetriever(secretStore); - this.endpointCertificateMetadata = params.endpointCertificateMetadata() - .or(() -> params.tlsSecretsKeyName().map(EndpointCertificateMetadataSerializer::fromString)); + this.endpointCertificateMetadata = params.endpointCertificateMetadata(); Optional<EndpointCertificateSecrets> endpointCertificateSecrets = endpointCertificateMetadata .or(() -> endpointCertificateMetadataStore.readEndpointCertificateMetadata(applicationId)) .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java index f3240a62133..13c88ece6d7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server.tenant; import com.yahoo.config.model.api.EndpointCertificateMetadata; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; +import com.yahoo.slime.Type; /** * (de)serializes endpoint certificate metadata @@ -30,26 +30,13 @@ public class EndpointCertificateMetadataSerializer { } public static EndpointCertificateMetadata fromSlime(Inspector inspector) { - switch (inspector.type()) { - case STRING: // TODO: Remove once all are transmitted and stored as JSON - return new EndpointCertificateMetadata( - inspector.asString() + "-key", - inspector.asString() + "-cert", - 0 - ); - case OBJECT: - return new EndpointCertificateMetadata( - inspector.field(keyNameField).asString(), - inspector.field(certNameField).asString(), - Math.toIntExact(inspector.field(versionField).asLong()) - ); - - default: - throw new IllegalArgumentException("Unknown format encountered for endpoint certificate metadata!"); + if (inspector.type() == Type.OBJECT) { + return new EndpointCertificateMetadata( + inspector.field(keyNameField).asString(), + inspector.field(certNameField).asString(), + Math.toIntExact(inspector.field(versionField).asLong()) + ); } - } - - public static EndpointCertificateMetadata fromString(String tlsSecretsKeys) { - return fromSlime(new Slime().setString(tlsSecretsKeys)); + throw new IllegalArgumentException("Unknown format encountered for endpoint certificate metadata!"); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java index 8e51ac424f9..415f0a41441 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java @@ -37,7 +37,7 @@ public class EndpointCertificateMetadataStore { EndpointCertificateMetadata endpointCertificateMetadata = EndpointCertificateMetadataSerializer.fromSlime(slime.get()); return Optional.of(endpointCertificateMetadata); } catch (Exception e) { - throw new RuntimeException("Error reading TLS secret key of " + application, e); + throw new RuntimeException("Error reading endpoint certificate metadata for " + application, e); } } @@ -48,17 +48,17 @@ public class EndpointCertificateMetadataStore { EndpointCertificateMetadataSerializer.toSlime(endpointCertificateMetadata, slime.setObject()); curator.set(endpointCertificateMetadataPathOf(application), SlimeUtils.toJsonBytes(slime)); } catch (Exception e) { - throw new RuntimeException("Could not write TLS secret key of " + application, e); + throw new RuntimeException("Could not write endpoint certificate metadata for " + application, e); } } - /** Returns a transaction which deletes these tls secrets key if they exist */ + /** Returns a transaction which deletes endpoint certificate metadata if it exists */ public CuratorTransaction delete(ApplicationId application) { if (!curator.exists(endpointCertificateMetadataPathOf(application))) return CuratorTransaction.empty(curator); return CuratorTransaction.from(CuratorOperations.delete(endpointCertificateMetadataPathOf(application).getAbsolute()), curator); } - /** Returns the path storing the tls secrets key for an application */ + /** Returns the path storing the endpoint certificate metadata for an application */ private Path endpointCertificateMetadataPathOf(ApplicationId application) { return path.append(application.serializedForm()); } diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index d301dffdf40..53cfa24b2ac 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -31,7 +31,6 @@ <component id="com.yahoo.vespa.config.server.version.VersionState" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.zookeeper.ConfigCurator" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.host.ConfigRequestHostLivenessTracker" bundle="configserver" /> - <component id="com.yahoo.container.jdisc.metric.state.StateMetricConsumerFactory" bundle="container-disc" /> <component id="com.yahoo.config.provision.Zone" bundle="config-provisioning" /> <component id="com.yahoo.vespa.config.server.application.ConfigConvergenceChecker" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.application.HttpProxy" bundle="configserver" /> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 4bb41a2c4de..4995dc0decc 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -289,27 +289,6 @@ public class SessionPreparerTest { } @Test - public void require_that_tlssecretkey_is_written() throws IOException { - var tlskey = "vespa.tlskeys.tenant1--app1"; - var applicationId = applicationId("test"); - var params = new PrepareParams.Builder().applicationId(applicationId).tlsSecretsKeyName(tlskey).build(); - - secretStore.put("vespa.tlskeys.tenant1--app1-cert", X509CertificateUtils.toPem(certificate)); - secretStore.put("vespa.tlskeys.tenant1--app1-key", KeyUtils.toPem(keyPair.getPrivate())); - - prepare(new File("src/test/resources/deploy/hosted-app"), params); - - // Read from zk and verify cert and key are available - Path tenantPath = TenantRepository.getTenantPath(applicationId.tenant()); - Optional<EndpointCertificateSecrets> endpointCertificateSecrets = new EndpointCertificateMetadataStore(curator, tenantPath) - .readEndpointCertificateMetadata(applicationId) - .flatMap(p -> new EndpointCertificateRetriever(secretStore).readEndpointCertificateSecrets(p)); - assertTrue(endpointCertificateSecrets.isPresent()); - assertTrue(endpointCertificateSecrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY")); - assertTrue(endpointCertificateSecrets.get().certificate().startsWith("-----BEGIN CERTIFICATE")); - } - - @Test public void require_that_endpoint_certificate_metadata_is_written() throws IOException { var applicationId = applicationId("test"); var params = new PrepareParams.Builder().applicationId(applicationId).endpointCertificateMetadata("{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 7}").build(); @@ -329,19 +308,18 @@ public class SessionPreparerTest { } @Test(expected = CertificateNotReadyException.class) - public void require_that_tlssecretkey_is_missing_when_not_in_secretstore() throws IOException { - var tlskey = "vespa.tlskeys.tenant1--app1"; + public void endpoint_certificate_is_missing_when_not_in_secretstore() throws IOException { var applicationId = applicationId("test"); - var params = new PrepareParams.Builder().applicationId(applicationId).tlsSecretsKeyName(tlskey).build(); + var params = new PrepareParams.Builder().applicationId(applicationId).endpointCertificateMetadata("{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 7}").build(); prepare(new File("src/test/resources/deploy/hosted-app"), params); } @Test(expected = CertificateNotReadyException.class) - public void require_that_tlssecretkey_is_missing_when_certificate_not_in_secretstore() throws IOException { + public void endpoint_certificate_is_missing_when_certificate_not_in_secretstore() throws IOException { var tlskey = "vespa.tlskeys.tenant1--app1"; var applicationId = applicationId("test"); - var params = new PrepareParams.Builder().applicationId(applicationId).tlsSecretsKeyName(tlskey).build(); - secretStore.put(tlskey+"-key", "KEY"); + var params = new PrepareParams.Builder().applicationId(applicationId).endpointCertificateMetadata("{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 7}").build(); + secretStore.put(tlskey+"-key", 7, "KEY"); prepare(new File("src/test/resources/deploy/hosted-app"), params); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java index 829e47bdb42..1a4c7c9669e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java @@ -53,18 +53,6 @@ public class EndpointCertificateMetadataStoreTest { } @Test - public void reads_string_format() { - curator.set(endpointCertificateMetadataPath, ("\"vespa.tlskeys.tenant1--app1\"").getBytes()); - - // Read from zk and verify cert and key are available - var endpointCertificateSecrets = endpointCertificateMetadataStore.readEndpointCertificateMetadata(applicationId) - .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets); - assertTrue(endpointCertificateSecrets.isPresent()); - assertTrue(endpointCertificateSecrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY")); - assertTrue(endpointCertificateSecrets.get().certificate().startsWith("-----BEGIN CERTIFICATE")); - } - - @Test public void reads_object_format() { curator.set(endpointCertificateMetadataPath, "{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 0}" |