diff options
77 files changed, 1283 insertions, 1038 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 2b55b1f1d10..66a23c79fbb 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -88,6 +88,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean allowMoreThanOneContentGroupDown = false; private boolean enableConditionalPutRemoveWriteRepair = false; private List<DataplaneToken> dataplaneTokens; + private boolean enableDataplaneProxy; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -148,6 +149,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean allowMoreThanOneContentGroupDown(ClusterSpec.Id id) { return allowMoreThanOneContentGroupDown; } @Override public boolean enableConditionalPutRemoveWriteRepair() { return enableConditionalPutRemoveWriteRepair; } @Override public List<DataplaneToken> dataplaneTokens() { return dataplaneTokens; } + @Override public boolean enableDataplaneProxy() { return enableDataplaneProxy; } public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim; @@ -393,6 +395,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setEnableDataplaneProxy(boolean enable) { + this.enableDataplaneProxy = enable; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java index ed53a1d2267..9b5a1429cb7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java @@ -15,7 +15,7 @@ import com.yahoo.vespa.model.container.component.UserBindingPattern; import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder; import org.w3c.dom.Element; -import java.util.OptionalInt; +import java.util.Collection; import java.util.Set; import static com.yahoo.vespa.model.container.ApplicationContainerCluster.METRICS_V2_HANDLER_BINDING_1; @@ -38,12 +38,9 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilderB VIP_HANDLER_BINDING); private final ApplicationContainerCluster cluster; - private final OptionalInt portBindingOverride; + private final Set<Integer> portBindingOverride; - public DomHandlerBuilder(ApplicationContainerCluster cluster) { - this(cluster, OptionalInt.empty()); - } - public DomHandlerBuilder(ApplicationContainerCluster cluster, OptionalInt portBindingOverride) { + public DomHandlerBuilder(ApplicationContainerCluster cluster, Set<Integer> portBindingOverride) { this.cluster = cluster; this.portBindingOverride = portBindingOverride; } @@ -51,23 +48,24 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilderB @Override protected Handler doBuild(DeployState deployState, TreeConfigProducer<AnyConfigProducer> parent, Element handlerElement) { Handler handler = createHandler(handlerElement); - OptionalInt port = portBindingOverride.isPresent() && deployState.isHosted() && deployState.featureFlags().useRestrictedDataPlaneBindings() - ? portBindingOverride - : OptionalInt.empty(); + var ports = deployState.isHosted() && deployState.featureFlags().useRestrictedDataPlaneBindings() + ? portBindingOverride : Set.<Integer>of(); - for (Element binding : XML.getChildren(handlerElement, "binding")) - addServerBinding(handler, userBindingPattern(XML.getValue(binding), port), deployState.getDeployLogger()); + for (Element xmlBinding : XML.getChildren(handlerElement, "binding")) + for (var binding : userBindingPattern(XML.getValue(xmlBinding), ports)) + addServerBinding(handler, binding, deployState.getDeployLogger()); DomComponentBuilder.addChildren(deployState, parent, handlerElement, handler); return handler; } - private static UserBindingPattern userBindingPattern(String path, OptionalInt port) { + private static Collection<UserBindingPattern> userBindingPattern(String path, Set<Integer> portBindingOverride) { UserBindingPattern bindingPattern = UserBindingPattern.fromPattern(path); - return port.isPresent() - ? bindingPattern.withPort(port.getAsInt()) - : bindingPattern; + if (portBindingOverride.isEmpty()) return Set.of(bindingPattern); + return portBindingOverride.stream() + .map(bindingPattern::withPort) + .toList(); } Handler createHandler(Element handlerElement) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java index 8163c268d09..a5a567b18f8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java @@ -14,7 +14,8 @@ import com.yahoo.vespa.model.container.component.UserBindingPattern; import java.nio.file.Path; import java.util.Collection; import java.util.Collections; -import java.util.OptionalInt; +import java.util.List; +import java.util.Set; /** * @author Einar M R Rosenvinge @@ -28,7 +29,7 @@ public class ContainerDocumentApi { private final boolean ignoreUndefinedFields; - public ContainerDocumentApi(ContainerCluster<?> cluster, HandlerOptions handlerOptions, boolean ignoreUndefinedFields, OptionalInt portOverride) { + public ContainerDocumentApi(ContainerCluster<?> cluster, HandlerOptions handlerOptions, boolean ignoreUndefinedFields, Set<Integer> portOverride) { this.ignoreUndefinedFields = ignoreUndefinedFields; addRestApiHandler(cluster, handlerOptions, portOverride); addFeedHandler(cluster, handlerOptions, portOverride); @@ -39,7 +40,7 @@ public class ContainerDocumentApi { c.addPlatformBundle(VESPACLIENT_CONTAINER_BUNDLE); } - private static void addFeedHandler(ContainerCluster<?> cluster, HandlerOptions handlerOptions, OptionalInt portOverride) { + private static void addFeedHandler(ContainerCluster<?> cluster, HandlerOptions handlerOptions, Set<Integer> portOverride) { String bindingSuffix = ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"; var executor = new Threadpool("feedapi-handler", handlerOptions.feedApiThreadpoolOptions); var handler = newVespaClientHandler("com.yahoo.vespa.http.server.FeedHandler", @@ -48,7 +49,7 @@ public class ContainerDocumentApi { } - private static void addRestApiHandler(ContainerCluster<?> cluster, HandlerOptions handlerOptions, OptionalInt portOverride) { + private static void addRestApiHandler(ContainerCluster<?> cluster, HandlerOptions handlerOptions, Set<Integer> portOverride) { var handler = newVespaClientHandler("com.yahoo.document.restapi.resource.DocumentV1ApiHandler", DOCUMENT_V1_PREFIX + "/*", handlerOptions, null, portOverride); cluster.addComponent(handler); @@ -65,34 +66,34 @@ public class ContainerDocumentApi { String bindingSuffix, HandlerOptions handlerOptions, Threadpool executor, - OptionalInt portOverride) { + Set<Integer> portOverride) { Handler handler = createHandler(componentId, executor); if (handlerOptions.bindings.isEmpty()) { - handler.addServerBindings( - bindingPattern(bindingSuffix, portOverride), - bindingPattern(bindingSuffix + '/', portOverride)); + handler.addServerBindings(bindingPattern(bindingSuffix, portOverride)); + handler.addServerBindings(bindingPattern(bindingSuffix + '/', portOverride)); } else { for (String rootBinding : handlerOptions.bindings) { String pathWithoutLeadingSlash = bindingSuffix.substring(1); - handler.addServerBindings( - userBindingPattern(rootBinding + pathWithoutLeadingSlash, portOverride), - userBindingPattern(rootBinding + pathWithoutLeadingSlash + '/', portOverride)); + handler.addServerBindings(userBindingPattern(rootBinding + pathWithoutLeadingSlash, portOverride)); + handler.addServerBindings(userBindingPattern(rootBinding + pathWithoutLeadingSlash + '/', portOverride)); } } return handler; } - private static BindingPattern bindingPattern(String path, OptionalInt port) { - return port.isPresent() - ? SystemBindingPattern.fromHttpPortAndPath(Integer.toString(port.getAsInt()), path) - : SystemBindingPattern.fromHttpPath(path); + private static List<BindingPattern> bindingPattern(String path, Set<Integer> ports) { + if (ports.isEmpty()) return List.of(SystemBindingPattern.fromHttpPath(path)); + return ports.stream() + .map(p -> (BindingPattern)SystemBindingPattern.fromHttpPortAndPath(p, path)) + .toList(); } - private static UserBindingPattern userBindingPattern(String path, OptionalInt port) { + private static List<BindingPattern> userBindingPattern(String path, Set<Integer> ports) { UserBindingPattern bindingPattern = UserBindingPattern.fromPattern(path); - return port.isPresent() - ? bindingPattern.withPort(port.getAsInt()) - : bindingPattern; + if (ports.isEmpty()) return List.of(bindingPattern); + return ports.stream() + .map(p -> (BindingPattern)bindingPattern.withPort(p)) + .toList(); } private static Handler createHandler(String className, Threadpool executor) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/DataplaneProxy.java b/config-model/src/main/java/com/yahoo/vespa/model/container/DataplaneProxy.java index fe7d9581e46..13aa65909bd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/DataplaneProxy.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/DataplaneProxy.java @@ -7,20 +7,23 @@ import com.yahoo.vespa.model.container.component.SimpleComponent; public class DataplaneProxy extends SimpleComponent implements DataplaneProxyConfig.Producer { - private final Integer port; + private final int mtlsPort; + private final int tokenPort; private final String serverCertificate; private final String serverKey; - public DataplaneProxy(Integer port, String serverCertificate, String serverKey) { + public DataplaneProxy(int mtlsPort, int tokenPort, String serverCertificate, String serverKey) { super(DataplaneProxyConfigurator.class.getName()); - this.port = port; + this.mtlsPort = mtlsPort; + this.tokenPort = tokenPort; this.serverCertificate = serverCertificate; this.serverKey = serverKey; } @Override public void getConfig(DataplaneProxyConfig.Builder builder) { - builder.port(port); + builder.mtlsPort(mtlsPort); + builder.tokenPort(tokenPort); builder.serverCertificate(serverCertificate); builder.serverKey(serverKey); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java index 9f2bfe9251b..31031aa5bf2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.model.container.ContainerThreadpool; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; @@ -51,6 +52,8 @@ public class Handler extends Component<Component<?, ?>, ComponentModel> { serverBindings.addAll(Arrays.asList(bindings)); } + public void addServerBindings(Collection<BindingPattern> bps) { serverBindings.addAll(bps); } + public void removeServerBinding(BindingPattern binding) { serverBindings.remove(binding); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SystemBindingPattern.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SystemBindingPattern.java index 606557670a5..0fb3ec389e0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SystemBindingPattern.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SystemBindingPattern.java @@ -15,6 +15,7 @@ public class SystemBindingPattern extends BindingPattern { public static SystemBindingPattern fromPattern(String binding) { return new SystemBindingPattern(binding);} public static SystemBindingPattern fromHttpPortAndPath(String port, String path) { return new SystemBindingPattern("http", "*", port, path); } public static SystemBindingPattern fromHttpPortAndPath(int port, String path) { return new SystemBindingPattern("http", "*", Integer.toString(port), path); } + public SystemBindingPattern withPort(int port) { return new SystemBindingPattern(scheme(), host(), Integer.toString(port), path()); } @Override public String toString() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java index 697cfc95039..4929c09d561 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.http.ssl.DefaultSslProvider; import com.yahoo.vespa.model.container.http.ssl.SslProvider; +import java.util.List; import java.util.Optional; /** @@ -40,6 +41,9 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig public void getConfig(ConnectorConfig.Builder connectorBuilder) { connectorBuilder.listenPort(listenPort); connectorBuilder.name(name); + connectorBuilder.accessLog(new ConnectorConfig.AccessLog.Builder() + .remoteAddressHeaders(List.of("x-forwarded-for")) + .remotePortHeaders(List.of("X-Forwarded-Port"))); sslProviderComponent.amendConnectorConfig(connectorBuilder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java index 6a2d9685a33..0388230fa6a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java @@ -63,17 +63,8 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro .searchHandlerPaths(List.of("/search")) ); if (isHostedVespa) { - // Proxy-protocol v1/v2 is used in hosted Vespa for remote address/port - builder.accessLog(new ServerConfig.AccessLog.Builder() - .remoteAddressHeaders(List.of()) - .remotePortHeaders(List.of())); - // Enable connection log hosted Vespa builder.connectionLog(new ServerConfig.ConnectionLog.Builder().enabled(true)); - } else { - builder.accessLog(new ServerConfig.AccessLog.Builder() - .remoteAddressHeaders(List.of("x-forwarded-for")) - .remotePortHeaders(List.of("X-Forwarded-Port"))); } configureJettyThreadpool(builder); builder.stopTimeout(300); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/CloudSslProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/CloudSslProvider.java index b231a4ad847..ab163719aac 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/CloudSslProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/CloudSslProvider.java @@ -2,8 +2,6 @@ package com.yahoo.vespa.model.container.http.ssl; import com.yahoo.jdisc.http.ConnectorConfig; -import com.yahoo.jdisc.http.ssl.impl.CloudSslContextProvider; -import com.yahoo.jdisc.http.ssl.impl.ConfiguredSslContextFactoryProvider; import java.util.Optional; @@ -16,10 +14,6 @@ import static com.yahoo.jdisc.http.ConnectorConfig.Ssl.ClientAuth; * @author andreer */ public class CloudSslProvider extends SslProvider { - public static final String COMPONENT_ID_PREFIX = "configured-ssl-provider@"; - public static final String MTLSONLY_COMPONENT_CLASS = ConfiguredSslContextFactoryProvider.class.getName(); - public static final String TOKEN_COMPONENT_CLASS = CloudSslContextProvider.class.getName(); - private final String privateKey; private final String certificate; private final String caCertificatePath; @@ -28,7 +22,7 @@ public class CloudSslProvider extends SslProvider { public CloudSslProvider(String servername, String privateKey, String certificate, String caCertificatePath, String caCertificate, ClientAuth.Enum clientAuthentication, boolean enableTokenSupport) { - super(COMPONENT_ID_PREFIX, servername, componentClass(enableTokenSupport), null); + super("cloud-ssl-provider@", servername, componentClass(enableTokenSupport), null); this.privateKey = privateKey; this.certificate = certificate; this.caCertificatePath = caCertificatePath; @@ -37,7 +31,9 @@ public class CloudSslProvider extends SslProvider { } private static String componentClass(boolean enableTokenSupport) { - return enableTokenSupport ? TOKEN_COMPONENT_CLASS : MTLSONLY_COMPONENT_CLASS; + return enableTokenSupport + ? "com.yahoo.jdisc.http.ssl.impl.CloudTokenSslContextProvider" + : "com.yahoo.jdisc.http.ssl.impl.ConfiguredSslContextFactoryProvider"; } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index 2b13cd21e99..cebe08288f6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -7,6 +7,7 @@ import com.yahoo.security.tls.TlsContext; import com.yahoo.vespa.model.container.http.ConnectorFactory; import java.time.Duration; +import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -22,6 +23,8 @@ public class HostedSslConnectorFactory extends ConnectorFactory { private final boolean proxyProtocolEnabled; private final boolean proxyProtocolMixedMode; private final Duration endpointConnectionTtl; + private final List<String> remoteAddressHeaders; + private final List<String> remotePortHeaders; public static Builder builder(String name, int listenPort) { return new Builder(name, listenPort); } @@ -32,6 +35,8 @@ public class HostedSslConnectorFactory extends ConnectorFactory { this.proxyProtocolEnabled = builder.proxyProtocolEnabled; this.proxyProtocolMixedMode = builder.proxyProtocolMixedMode; this.endpointConnectionTtl = builder.endpointConnectionTtl; + this.remoteAddressHeaders = List.copyOf(builder.remoteAddressHeaders); + this.remotePortHeaders = List.copyOf(builder.remotePortHeaders); } private static SslProvider createSslProvider(Builder builder) { @@ -62,15 +67,21 @@ public class HostedSslConnectorFactory extends ConnectorFactory { .proxyProtocol(new ConnectorConfig.ProxyProtocol.Builder() .enabled(proxyProtocolEnabled).mixedMode(proxyProtocolMixedMode)) .idleTimeout(Duration.ofSeconds(30).toSeconds()) - .maxConnectionLife(endpointConnectionTtl != null ? endpointConnectionTtl.toSeconds() : 0); + .maxConnectionLife(endpointConnectionTtl != null ? endpointConnectionTtl.toSeconds() : 0) + .accessLog(new ConnectorConfig.AccessLog.Builder() + .remoteAddressHeaders(remoteAddressHeaders) + .remotePortHeaders(remotePortHeaders)); + } public enum SslClientAuth { WANT, NEED, WANT_WITH_ENFORCER } public static class Builder { final String name; final int port; + final List<String> remoteAddressHeaders = new ArrayList<>(); + final List<String> remotePortHeaders = new ArrayList<>(); SslClientAuth clientAuth; - List<String> tlsCiphersOverride; + List<String> tlsCiphersOverride = List.of(); boolean proxyProtocolEnabled; boolean proxyProtocolMixedMode; Duration endpointConnectionTtl; @@ -88,6 +99,8 @@ public class HostedSslConnectorFactory extends ConnectorFactory { public Builder tlsCaCertificatesPath(String path) { this.tlsCaCertificatesPath = path; return this; } public Builder tlsCaCertificatesPem(String pem) { this.tlsCaCertificatesPem = pem; return this; } public Builder tokenEndpoint(boolean enable) { this.tokenEndpoint = enable; return this; } + public Builder remoteAddressHeader(String header) { this.remoteAddressHeaders.add(header); return this; } + public Builder remotePortHeader(String header) { this.remotePortHeaders.add(header); return this; } public HostedSslConnectorFactory build() { return new HostedSslConnectorFactory(this); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java index 330e1f96dc7..b05466d54ab 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java @@ -6,6 +6,8 @@ import com.yahoo.vespa.model.container.component.BindingPattern; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.chain.Chains; +import java.util.List; + /** * Root config producer for processing * @@ -13,7 +15,7 @@ import com.yahoo.vespa.model.container.component.chain.Chains; */ public class ProcessingChains extends Chains<ProcessingChain> { - public static final BindingPattern[] defaultBindings = new BindingPattern[]{SystemBindingPattern.fromHttpPath("/processing/*")}; + public static final List<BindingPattern> defaultBindings = List.of(SystemBindingPattern.fromHttpPath("/processing/*")); public ProcessingChains(TreeConfigProducer<? super Chains> parent, String subId) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudDataPlaneFilter.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudDataPlaneFilter.java index efa5ee01506..2d0d47288d1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudDataPlaneFilter.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudDataPlaneFilter.java @@ -5,7 +5,6 @@ import com.yahoo.component.ComponentSpecification; import com.yahoo.component.chain.dependencies.Dependencies; import com.yahoo.component.chain.model.ChainedComponentModel; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.provision.DataplaneToken; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.jdisc.http.filter.security.cloud.config.CloudDataPlaneFilterConfig; import com.yahoo.security.X509CertificateUtils; @@ -13,7 +12,6 @@ import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.http.Client; import com.yahoo.vespa.model.container.http.Filter; -import java.time.Instant; import java.util.Collection; import java.util.List; @@ -24,15 +22,11 @@ class CloudDataPlaneFilter extends Filter implements CloudDataPlaneFilterConfig. private final Collection<Client> clients; private final boolean clientsLegacyMode; - private final String tokenContext; CloudDataPlaneFilter(ApplicationContainerCluster cluster, DeployState state) { super(model()); this.clients = List.copyOf(cluster.getClients()); this.clientsLegacyMode = cluster.clientsLegacyMode(); - // Token domain must be identical to the domain used for generating the tokens - this.tokenContext = "Vespa Cloud tenant data plane:%s" - .formatted(state.getProperties().applicationId().tenant().value()); } private static ChainedComponentModel model() { @@ -51,21 +45,11 @@ class CloudDataPlaneFilter extends Filter implements CloudDataPlaneFilterConfig. .map(x -> new CloudDataPlaneFilterConfig.Clients.Builder() .id(x.id()) .certificates(x.certificates().stream().map(X509CertificateUtils::toPem).toList()) - .tokens(tokensConfig(x.tokens())) .permissions(x.permissions())) .toList(); - builder.clients(clientsCfg).legacyMode(false).tokenContext(tokenContext); + builder.clients(clientsCfg).legacyMode(false); } } - private static List<CloudDataPlaneFilterConfig.Clients.Tokens.Builder> tokensConfig(Collection<DataplaneToken> tokens) { - return tokens.stream() - .map(token -> new CloudDataPlaneFilterConfig.Clients.Tokens.Builder() - .id(token.tokenId()) - .fingerprints(token.versions().stream().map(DataplaneToken.Version::fingerprint).toList()) - .checkAccessHashes(token.versions().stream().map(DataplaneToken.Version::checkAccessHash).toList()) - .expirations(token.versions().stream().map(v -> v.expiration().map(Instant::toString).orElse("<none>")).toList())) - .toList(); - } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudTokenDataPlaneFilter.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudTokenDataPlaneFilter.java new file mode 100644 index 00000000000..5b57682e759 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudTokenDataPlaneFilter.java @@ -0,0 +1,61 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.dependencies.Dependencies; +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.provision.DataplaneToken; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.jdisc.http.filter.security.cloud.config.CloudTokenDataPlaneFilterConfig; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; +import com.yahoo.vespa.model.container.http.Client; +import com.yahoo.vespa.model.container.http.Filter; + +import java.time.Instant; +import java.util.Collection; +import java.util.List; + +class CloudTokenDataPlaneFilter extends Filter implements CloudTokenDataPlaneFilterConfig.Producer { + private final Collection<Client> clients; + private final String tokenContext; + + CloudTokenDataPlaneFilter(ApplicationContainerCluster cluster, DeployState state) { + super(model()); + this.clients = List.copyOf(cluster.getClients()); + // Token domain must be identical to the domain used for generating the tokens + this.tokenContext = "Vespa Cloud tenant data plane:%s" + .formatted(state.getProperties().applicationId().tenant().value()); + } + + private static ChainedComponentModel model() { + return new ChainedComponentModel( + new BundleInstantiationSpecification( + new ComponentSpecification("com.yahoo.jdisc.http.filter.security.cloud.CloudTokenDataPlaneFilter"), + null, + new ComponentSpecification("jdisc-security-filters")), + Dependencies.emptyDependencies()); + } + + @Override + public void getConfig(CloudTokenDataPlaneFilterConfig.Builder builder) { + var clientsCfg = clients.stream() + .map(x -> new CloudTokenDataPlaneFilterConfig.Clients.Builder() + .id(x.id()) + .tokens(tokensConfig(x.tokens())) + .permissions(x.permissions())) + .toList(); + builder.clients(clientsCfg).tokenContext(tokenContext); + } + + private static List<CloudTokenDataPlaneFilterConfig.Clients.Tokens.Builder> tokensConfig(Collection<DataplaneToken> tokens) { + return tokens.stream() + .map(token -> new CloudTokenDataPlaneFilterConfig.Clients.Tokens.Builder() + .id(token.tokenId()) + .fingerprints(token.versions().stream().map(DataplaneToken.Version::fingerprint).toList()) + .checkAccessHashes(token.versions().stream().map(DataplaneToken.Version::checkAccessHash).toList()) + .expirations(token.versions().stream().map(v -> v.expiration().map(Instant::toString).orElse("<none>")).toList())) + .toList(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index c97ea6671e8..1036a615bb5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -139,9 +139,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // Default path to vip status file for container in Hosted Vespa. static final String HOSTED_VESPA_STATUS_FILE = Defaults.getDefaults().underVespaHome("var/vespa/load-balancer/status.html"); - // Data plane port for hosted Vespa - public static final int HOSTED_VESPA_DATAPLANE_PORT = 4443; - //Path to vip status file for container in Hosted Vespa. Only used if set, else use HOSTED_VESPA_STATUS_FILE private static final String HOSTED_VESPA_STATUS_FILE_SETTING = "VESPA_LB_STATUS_FILE"; @@ -461,15 +458,16 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addHostedImplicitHttpIfNotPresent(deployState, cluster); addHostedImplicitAccessControlIfNotPresent(deployState, cluster); addDefaultConnectorHostedFilterBinding(cluster); - addAdditionalHostedConnector(deployState, cluster); + addCloudMtlsConnector(deployState, cluster); addCloudDataPlaneFilter(deployState, cluster); + addCloudTokenSupport(deployState, cluster); } } private static void addCloudDataPlaneFilter(DeployState deployState, ApplicationContainerCluster cluster) { if (!deployState.isHosted() || !deployState.zone().system().isPublic()) return; - var dataplanePort = getDataplanePort(deployState); + var dataplanePort = getMtlsDataplanePort(deployState); // Setup secure filter chain var secureChain = new HttpFilterChain("cloud-data-plane-secure", HttpFilterChain.Type.SYSTEM); secureChain.addInnerComponent(new CloudDataPlaneFilter(cluster, deployState)); @@ -599,12 +597,12 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .ifPresent(accessControl -> accessControl.configureDefaultHostedConnector(cluster.getHttp())); ; } - private void addAdditionalHostedConnector(DeployState state, ApplicationContainerCluster cluster) { + private void addCloudMtlsConnector(DeployState state, ApplicationContainerCluster cluster) { JettyHttpServer server = cluster.getHttp().getHttpServer().get(); String serverName = server.getComponentId().getName(); // If the deployment contains certificate/private key reference, setup TLS port - var builder = HostedSslConnectorFactory.builder(serverName, getDataplanePort(state)) + var builder = HostedSslConnectorFactory.builder(serverName, getMtlsDataplanePort(state)) .proxyProtocol(true, state.getProperties().featureFlags().enableProxyProtocolMixedMode()) .tlsCiphersOverride(state.getProperties().tlsCiphersOverride()) .endpointConnectionTtl(state.getProperties().endpointConnectionTtl()); @@ -627,22 +625,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .orElse(false); builder.clientAuth(needAuth ? SslClientAuth.NEED : SslClientAuth.WANT); } - - boolean enableTokenSupport = state.featureFlags().enableDataplaneProxy() - && cluster.getClients().stream().anyMatch(c -> !c.tokens().isEmpty()); - - // Set up component to generate proxy cert if token support is enabled - if (enableTokenSupport) { - cluster.addSimpleComponent(DataplaneProxyCredentials.class); - cluster.addSimpleComponent(DataplaneProxyService.class); - - var dataplaneProxy = new DataplaneProxy( - getDataplanePort(state), - endpointCert.certificate(), - endpointCert.key()); - cluster.addComponent(dataplaneProxy); - builder.tokenEndpoint(true); - } } else { builder.clientAuth(SslClientAuth.WANT_WITH_ENFORCER); } @@ -651,6 +633,47 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { server.addConnector(connectorFactory); } + private void addCloudTokenSupport(DeployState state, ApplicationContainerCluster cluster) { + var server = cluster.getHttp().getHttpServer().get(); + boolean enableTokenSupport = state.isHosted() && state.zone().system().isPublic() + && state.featureFlags().enableDataplaneProxy() + && cluster.getClients().stream().anyMatch(c -> !c.tokens().isEmpty()); + if (!enableTokenSupport) return; + var endpointCert = state.endpointCertificateSecrets().orElseThrow(); + int tokenPort = getTokenDataplanePort(state).orElseThrow(); + + // Set up component to generate proxy cert if token support is enabled + cluster.addSimpleComponent(DataplaneProxyCredentials.class); + cluster.addSimpleComponent(DataplaneProxyService.class); + var dataplaneProxy = new DataplaneProxy( + getMtlsDataplanePort(state), + tokenPort, + endpointCert.certificate(), + endpointCert.key()); + cluster.addComponent(dataplaneProxy); + + // Setup dedicated connector + var connector = HostedSslConnectorFactory.builder(server.getComponentId().getName()+"-token", tokenPort) + .tokenEndpoint(true) + .proxyProtocol(false, false) + .endpointCertificate(endpointCert) + .remoteAddressHeader("X-Forwarded-For") + .remotePortHeader("X-Forwarded-Port") + .clientAuth(SslClientAuth.NEED) + .build(); + server.addConnector(connector); + + // Setup token filter chain + var tokenChain = new HttpFilterChain("cloud-token-data-plane-secure", HttpFilterChain.Type.SYSTEM); + tokenChain.addInnerComponent(new CloudTokenDataPlaneFilter(cluster, state)); + cluster.getHttp().getFilterChains().add(tokenChain); + + // Set as default filter for token port + cluster.getHttp().getHttpServer().orElseThrow().getConnectorFactories().stream() + .filter(c -> c.getListenPort() == tokenPort).findAny().orElseThrow() + .setDefaultRequestFilterChain(tokenChain.getComponentId()); + } + // Returns the client certificates of the clients defined for an application cluster private List<X509Certificate> getClientCertificates(ApplicationContainerCluster cluster) { return cluster.getClients() @@ -810,7 +833,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } private void addUserHandlers(DeployState deployState, ApplicationContainerCluster cluster, Element spec, ConfigModelContext context) { - OptionalInt portBindingOverride = isHostedTenantApplication(context) ? OptionalInt.of(getDataplanePort(deployState)) : OptionalInt.empty(); + var portBindingOverride = isHostedTenantApplication(context) ? getDataplanePorts(deployState) : Set.<Integer>of(); for (Element component: XML.getChildren(spec, "handler")) { cluster.addComponent( new DomHandlerBuilder(cluster, portBindingOverride).build(deployState, cluster, component)); @@ -1099,12 +1122,12 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } private void addSearchHandler(DeployState deployState, ApplicationContainerCluster cluster, Element searchElement, ConfigModelContext context) { - BindingPattern bindingPattern = SearchHandler.DEFAULT_BINDING; + var bindingPatterns = List.<BindingPattern>of(SearchHandler.DEFAULT_BINDING); if (isHostedTenantApplication(context) && deployState.featureFlags().useRestrictedDataPlaneBindings()) { - bindingPattern = SearchHandler.bindingPattern(Optional.of(Integer.toString(getDataplanePort(deployState)))); + bindingPatterns = SearchHandler.bindingPattern(getDataplanePorts(deployState)); } SearchHandler searchHandler = new SearchHandler(cluster, - serverBindings(deployState, context, searchElement, bindingPattern), + serverBindings(deployState, context, searchElement, bindingPatterns), ContainerThreadpool.UserOptions.fromXml(searchElement).orElse(null)); cluster.addComponent(searchHandler); @@ -1112,41 +1135,43 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { searchHandler.addComponent(Component.fromClassAndBundle(SearchHandler.EXECUTION_FACTORY, PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE)); } - private List<BindingPattern> serverBindings(DeployState deployState, ConfigModelContext context, Element searchElement, BindingPattern... defaultBindings) { + private List<BindingPattern> serverBindings(DeployState deployState, ConfigModelContext context, Element searchElement, Collection<BindingPattern> defaultBindings) { List<Element> bindings = XML.getChildren(searchElement, "binding"); if (bindings.isEmpty()) - return List.of(defaultBindings); + return List.copyOf(defaultBindings); return toBindingList(deployState, context, bindings); } private List<BindingPattern> toBindingList(DeployState deployState, ConfigModelContext context, List<Element> bindingElements) { List<BindingPattern> result = new ArrayList<>(); - OptionalInt portOverride = isHostedTenantApplication(context) && deployState.featureFlags().useRestrictedDataPlaneBindings() ? OptionalInt.of(getDataplanePort(deployState)) : OptionalInt.empty(); + var portOverride = isHostedTenantApplication(context) && deployState.featureFlags().useRestrictedDataPlaneBindings() ? getDataplanePorts(deployState) : Set.<Integer>of(); for (Element element: bindingElements) { String text = element.getTextContent().trim(); if (!text.isEmpty()) - result.add(userBindingPattern(text, portOverride)); + result.addAll(userBindingPattern(text, portOverride)); } return result; } - private static UserBindingPattern userBindingPattern(String path, OptionalInt portOverride) { + private static Collection<UserBindingPattern> userBindingPattern(String path, Set<Integer> portBindingOverride) { UserBindingPattern bindingPattern = UserBindingPattern.fromPattern(path); - return portOverride.isPresent() - ? bindingPattern.withPort(portOverride.getAsInt()) - : bindingPattern; + if (portBindingOverride.isEmpty()) return Set.of(bindingPattern); + return portBindingOverride.stream() + .map(bindingPattern::withPort) + .toList(); } + private ContainerDocumentApi buildDocumentApi(DeployState deployState, ApplicationContainerCluster cluster, Element spec, ConfigModelContext context) { Element documentApiElement = XML.getChild(spec, "document-api"); if (documentApiElement == null) return null; ContainerDocumentApi.HandlerOptions documentApiOptions = DocumentApiOptionsBuilder.build(documentApiElement); Element ignoreUndefinedFields = XML.getChild(documentApiElement, "ignore-undefined-fields"); - OptionalInt portBindingOverride = deployState.featureFlags().useRestrictedDataPlaneBindings() && isHostedTenantApplication(context) - ? OptionalInt.of(getDataplanePort(deployState)) - : OptionalInt.empty(); + var portBindingOverride = deployState.featureFlags().useRestrictedDataPlaneBindings() && isHostedTenantApplication(context) + ? getDataplanePorts(deployState) + : Set.<Integer>of(); return new ContainerDocumentApi(cluster, documentApiOptions, "true".equals(XML.getValue(ignoreUndefinedFields)), portBindingOverride); } @@ -1406,8 +1431,18 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } - private static int getDataplanePort(DeployState deployState) { - return deployState.featureFlags().enableDataplaneProxy() ? 8443 : HOSTED_VESPA_DATAPLANE_PORT; + private static Set<Integer> getDataplanePorts(DeployState ds) { + var tokenPort = getTokenDataplanePort(ds); + var mtlsPort = getMtlsDataplanePort(ds); + return tokenPort.isPresent() ? Set.of(mtlsPort, tokenPort.getAsInt()) : Set.of(mtlsPort); + } + + private static int getMtlsDataplanePort(DeployState ds) { + return ds.featureFlags().enableDataplaneProxy() ? 8443 : 4443; + } + + private static OptionalInt getTokenDataplanePort(DeployState ds) { + return ds.featureFlags().enableDataplaneProxy() ? OptionalInt.of(8444) : OptionalInt.empty(); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java index 96f653bf793..f3e02adff6b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java @@ -38,9 +38,18 @@ public class ModelIdResolver { models.put("multilingual-e5-base", "https://data.vespa.oath.cloud/onnx_models/multilingual-e5-base/model.onnx"); models.put("multilingual-e5-base-vocab", "https://data.vespa.oath.cloud/onnx_models/multilingual-e5-base/tokenizer.json"); + models.put("multilingual-e5-small", "https://data.vespa.oath.cloud/onnx_models/multilingual-e5-small/model.onnx"); + models.put("multilingual-e5-small-vocab", "https://data.vespa.oath.cloud/onnx_models/multilingual-e5-small/tokenizer.json"); + + models.put("multilingual-e5-small-cpu-friendly", "https://data.vespa.oath.cloud/onnx_models/multilingual-e5-small-cpu-friendly/model.onnx"); + models.put("multilingual-e5-small-vocab-cpu-friendly", "https://data.vespa.oath.cloud/onnx_models/multilingual-e5-small-cpu-friendly/tokenizer.json"); + models.put("e5-small-v2", "https://data.vespa.oath.cloud/onnx_models/e5-small-v2/model.onnx"); models.put("e5-small-v2-vocab", "https://data.vespa.oath.cloud/onnx_models/e5-small-v2/tokenizer.json"); + models.put("e5-small-v2-cpu-friendly", "https://data.vespa.oath.cloud/onnx_models/e5-small-v2-cpu-friendly/model.onnx"); + models.put("e5-small-v2-vocab-cpu-friendly", "https://data.vespa.oath.cloud/onnx_models/e5-small-v2-cpu-friendly/tokenizer.json"); + models.put("e5-base-v2", "https://data.vespa.oath.cloud/onnx_models/e5-base-v2/model.onnx"); models.put("e5-base-v2-vocab", "https://data.vespa.oath.cloud/onnx_models/e5-base-v2/tokenizer.json"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java index ebb22b2b73b..6cfef153fee 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java @@ -10,8 +10,8 @@ import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; +import java.util.Collection; import java.util.List; -import java.util.Optional; import static com.yahoo.container.bundle.BundleInstantiationSpecification.fromSearchAndDocproc; @@ -28,7 +28,7 @@ class SearchHandler extends ProcessingHandler<SearchChains> { static final String EXECUTION_FACTORY_CLASSNAME = EXECUTION_FACTORY.getName(); static final BundleInstantiationSpecification HANDLER_SPEC = fromSearchAndDocproc(HANDLER_CLASSNAME); - static final BindingPattern DEFAULT_BINDING = bindingPattern(Optional.empty()); + static final BindingPattern DEFAULT_BINDING = SystemBindingPattern.fromHttpPath("/search/*"); SearchHandler(ApplicationContainerCluster cluster, List<BindingPattern> bindings, @@ -37,12 +37,11 @@ class SearchHandler extends ProcessingHandler<SearchChains> { bindings.forEach(this::addServerBindings); } - static BindingPattern bindingPattern(Optional<String> port) { - String path = "/search/*"; - return port - .filter(s -> !s.isBlank()) - .map(s -> SystemBindingPattern.fromHttpPortAndPath(s, path)) - .orElseGet(() -> SystemBindingPattern.fromHttpPath(path)); + static List<BindingPattern> bindingPattern(Collection<Integer> ports) { + if (ports.isEmpty()) return List.of(DEFAULT_BINDING); + return ports.stream() + .map(s -> (BindingPattern)SystemBindingPattern.fromHttpPortAndPath(s, DEFAULT_BINDING.path())) + .toList(); } private static class Threadpool extends ContainerThreadpool { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/CloudDataPlaneFilterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/CloudDataPlaneFilterTest.java index 02ff7b8a03f..94d92b355f9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/CloudDataPlaneFilterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/CloudDataPlaneFilterTest.java @@ -6,7 +6,6 @@ import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.config.provision.DataplaneToken; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; @@ -35,17 +34,14 @@ import java.nio.file.Files; import java.nio.file.Path; import java.security.KeyPair; import java.security.cert.X509Certificate; -import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.Collection; import java.util.List; import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertIterableEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -89,7 +85,6 @@ public class CloudDataPlaneFilterTest extends ContainerModelBuilderTestBase { CloudDataPlaneFilterConfig.Clients client = clients.get(0); assertEquals("foo", client.id()); assertIterableEquals(List.of("read", "write"), client.permissions()); - assertTrue(client.tokens().isEmpty()); assertIterableEquals(List.of(X509CertificateUtils.toPem(certificate)), client.certificates()); ConnectorConfig connectorConfig = connectorConfig(); @@ -123,43 +118,6 @@ public class CloudDataPlaneFilterTest extends ContainerModelBuilderTestBase { } @Test - void generates_correct_config_for_tokens() throws IOException { - var certFile = securityFolder.resolve("foo.pem"); - var clusterElem = DomBuilderTest.parse( - """ - <container version='1.0'> - <clients> - <client id="foo" permissions="read,write"> - <certificate file="%s"/> - </client> - <client id="bar" permissions="read"> - <token id="my-token"/> - </client> - </clients> - </container> - """ - .formatted(applicationFolder.toPath().relativize(certFile).toString())); - createCertificate(certFile); - buildModel(clusterElem); - - var cfg = root.getConfig(CloudDataPlaneFilterConfig.class, cloudDataPlaneFilterConfigId); - var tokenClient = cfg.clients().stream().filter(c -> c.id().equals("bar")).findAny().orElse(null); - assertNotNull(tokenClient); - assertEquals(List.of("read"), tokenClient.permissions()); - assertTrue(tokenClient.certificates().isEmpty()); - var expectedTokenCfg = tokenConfig( - "my-token", List.of("myfingerprint1", "myfingerprint2"), List.of("myaccesshash1", "myaccesshash2"), - List.of("<none>", "2243-10-17T00:00:00Z")); - assertEquals(List.of(expectedTokenCfg), tokenClient.tokens()); - } - - private static CloudDataPlaneFilterConfig.Clients.Tokens tokenConfig( - String id, Collection<String> fingerprints, Collection<String> accessCheckHashes, Collection<String> expirations) { - return new CloudDataPlaneFilterConfig.Clients.Tokens.Builder() - .id(id).fingerprints(fingerprints).checkAccessHashes(accessCheckHashes).expirations(expirations).build(); - } - - @Test public void it_rejects_files_without_certificates() throws IOException { Path certFile = securityFolder.resolve("foo.pem"); Element clusterElem = DomBuilderTest.parse( @@ -231,9 +189,6 @@ public class CloudDataPlaneFilterTest extends ContainerModelBuilderTestBase { .properties( new TestProperties() .setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY"))) - .setDataplaneTokens(List.of(new DataplaneToken("my-token", List.of( - new DataplaneToken.Version("myfingerprint1", "myaccesshash1", Optional.empty()), - new DataplaneToken.Version("myfingerprint2", "myaccesshash2", Optional.of(Instant.EPOCH.plus(Duration.ofDays(100000)))))))) .setHostedVespa(true)) .zone(new Zone(SystemName.PublicCd, Environment.dev, RegionName.defaultName())) .build(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/CloudTokenDataPlaneFilterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/CloudTokenDataPlaneFilterTest.java new file mode 100644 index 00000000000..15e1d61c951 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/CloudTokenDataPlaneFilterTest.java @@ -0,0 +1,105 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.DataplaneToken; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; +import com.yahoo.jdisc.http.filter.security.cloud.config.CloudTokenDataPlaneFilterConfig; +import com.yahoo.vespa.model.container.ContainerModel; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.w3c.dom.Element; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +import static com.yahoo.vespa.model.container.xml.CloudDataPlaneFilterTest.createCertificate; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class CloudTokenDataPlaneFilterTest extends ContainerModelBuilderTestBase { + + @TempDir + public File applicationFolder; + + Path securityFolder; + private static final String filterConfigId = "container/filters/chain/cloud-token-data-plane-secure/component/" + + "com.yahoo.jdisc.http.filter.security.cloud.CloudTokenDataPlaneFilter"; + + @BeforeEach + public void setup() throws IOException { + securityFolder = applicationFolder.toPath().resolve("security"); + Files.createDirectories(securityFolder); + } + + @Test + void generates_correct_config_for_tokens() throws IOException { + var certFile = securityFolder.resolve("foo.pem"); + var clusterElem = DomBuilderTest.parse( + """ + <container version='1.0'> + <clients> + <client id="foo" permissions="read,write"> + <certificate file="%s"/> + </client> + <client id="bar" permissions="read"> + <token id="my-token"/> + </client> + </clients> + </container> + """ + .formatted(applicationFolder.toPath().relativize(certFile).toString())); + createCertificate(certFile); + buildModel(clusterElem); + + var cfg = root.getConfig(CloudTokenDataPlaneFilterConfig.class, filterConfigId); + var tokenClient = cfg.clients().stream().filter(c -> c.id().equals("bar")).findAny().orElse(null); + assertNotNull(tokenClient); + assertEquals(List.of("read"), tokenClient.permissions()); + var expectedTokenCfg = tokenConfig( + "my-token", List.of("myfingerprint1", "myfingerprint2"), List.of("myaccesshash1", "myaccesshash2"), + List.of("<none>", "2243-10-17T00:00:00Z")); + assertEquals(List.of(expectedTokenCfg), tokenClient.tokens()); + } + + private static CloudTokenDataPlaneFilterConfig.Clients.Tokens tokenConfig( + String id, Collection<String> fingerprints, Collection<String> accessCheckHashes, Collection<String> expirations) { + return new CloudTokenDataPlaneFilterConfig.Clients.Tokens.Builder() + .id(id).fingerprints(fingerprints).checkAccessHashes(accessCheckHashes).expirations(expirations).build(); + } + + public List<ContainerModel> buildModel(Element... clusterElem) { + var applicationPackage = new MockApplicationPackage.Builder() + .withRoot(applicationFolder) + .build(); + + DeployState state = new DeployState.Builder() + .applicationPackage(applicationPackage) + .properties( + new TestProperties() + .setEnableDataplaneProxy(true) + .setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY"))) + .setDataplaneTokens(List.of(new DataplaneToken("my-token", List.of( + new DataplaneToken.Version("myfingerprint1", "myaccesshash1", Optional.empty()), + new DataplaneToken.Version("myfingerprint2", "myaccesshash2", Optional.of(Instant.EPOCH.plus(Duration.ofDays(100000)))))))) + .setHostedVespa(true)) + .zone(new Zone(SystemName.PublicCd, Environment.dev, RegionName.defaultName())) + .build(); + return createModel(root, state, null, clusterElem); + } +} diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt index 85fc1158afe..29ed0f53421 100644 --- a/configdefinitions/src/vespa/CMakeLists.txt +++ b/configdefinitions/src/vespa/CMakeLists.txt @@ -89,3 +89,4 @@ install_config_definition(hugging-face-embedder.def embedding.huggingface.huggin install_config_definition(hugging-face-tokenizer.def language.huggingface.config.hugging-face-tokenizer.def) install_config_definition(bert-base-embedder.def embedding.bert-base-embedder.def) install_config_definition(cloud-data-plane-filter.def jdisc.http.filter.security.cloud.config.cloud-data-plane-filter.def) +install_config_definition(cloud-token-data-plane-filter.def jdisc.http.filter.security.cloud.config.cloud-token-data-plane-filter.def) diff --git a/configdefinitions/src/vespa/cloud-data-plane-filter.def b/configdefinitions/src/vespa/cloud-data-plane-filter.def index d73c5a49c81..47478a28039 100644 --- a/configdefinitions/src/vespa/cloud-data-plane-filter.def +++ b/configdefinitions/src/vespa/cloud-data-plane-filter.def @@ -2,11 +2,6 @@ namespace=jdisc.http.filter.security.cloud.config legacyMode bool default=false -tokenContext string default="" clients[].id string clients[].permissions[] string clients[].certificates[] string -clients[].tokens[].id string -clients[].tokens[].fingerprints[] string -clients[].tokens[].checkAccessHashes[] string -clients[].tokens[].expirations[] string diff --git a/configdefinitions/src/vespa/cloud-token-data-plane-filter.def b/configdefinitions/src/vespa/cloud-token-data-plane-filter.def new file mode 100644 index 00000000000..3219ae4fa48 --- /dev/null +++ b/configdefinitions/src/vespa/cloud-token-data-plane-filter.def @@ -0,0 +1,10 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=jdisc.http.filter.security.cloud.config + +tokenContext string default="" +clients[].id string +clients[].permissions[] string +clients[].tokens[].id string +clients[].tokens[].fingerprints[] string +clients[].tokens[].checkAccessHashes[] string +clients[].tokens[].expirations[] string diff --git a/configdefinitions/src/vespa/dataplane-proxy.def b/configdefinitions/src/vespa/dataplane-proxy.def index 9ce3e4b4b7b..dd1d734a91c 100644 --- a/configdefinitions/src/vespa/dataplane-proxy.def +++ b/configdefinitions/src/vespa/dataplane-proxy.def @@ -2,7 +2,8 @@ namespace=cloud.config # The port Jdisc will be listening on -port int +tokenPort int +mtlsPort int # Server certificate and key to be used when creating server socket serverCertificate string diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java index b2762b2a3d4..b33b21691af 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java @@ -98,8 +98,8 @@ public class ApplicationApiHandler extends SessionHandler { "Unable to parse multipart in deploy from tenant '" + tenantName.value() + "': " + Exceptions.toMessageString(e)); var message = "Deploy request from '" + tenantName.value() + "' contains invalid data: " + e.getMessage(); - log.log(FINE, message + ", parts: " + parts, e); - throw new BadRequestException("Deploy request from '" + tenantName.value() + "' contains invalid data: " + e.getMessage()); + log.log(INFO, message + ", parts: " + parts, e); + throw new BadRequestException(message); } } else { prepareParams = PrepareParams.fromHttpRequest(request, tenantName, zookeeperBarrierTimeout); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java index b627fe9ba3b..df1fdddf409 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java @@ -121,7 +121,7 @@ public abstract class Session implements Comparable<Session> { } void setApplicationPackageReference(FileReference applicationPackageReference) { - sessionZooKeeperClient.writeApplicationPackageReference(Optional.ofNullable(applicationPackageReference)); + sessionZooKeeperClient.writeApplicationPackageReference(applicationPackageReference); } public void setVespaVersion(Version version) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index ae87a0dd182..94745be9d2a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -394,7 +394,7 @@ public class SessionPreparer { zkDeployer.deploy(applicationPackage, fileRegistryMap, allocatedHosts); // Note: When changing the below you need to also change similar calls in SessionRepository.createSessionFromExisting() zooKeeperClient.writeApplicationId(applicationId); - zooKeeperClient.writeApplicationPackageReference(Optional.of(fileReference)); + zooKeeperClient.writeApplicationPackageReference(fileReference); zooKeeperClient.writeVespaVersion(vespaVersion); zooKeeperClient.writeDockerImageRepository(dockerImageRepository); zooKeeperClient.writeAthenzDomain(athenzDomain); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index 23b6fe075fa..e2e5e2bcce4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; -import com.yahoo.component.Vtag; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; @@ -38,6 +37,7 @@ import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; import org.apache.zookeeper.data.Stat; + import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; @@ -175,16 +175,14 @@ public class SessionZooKeeperClient { .orElseThrow(() -> new NotFoundException("Could not find application id for session " + sessionId)); } - void writeApplicationPackageReference(Optional<FileReference> applicationPackageReference) { - applicationPackageReference.ifPresent( - reference -> curator.set(applicationPackageReferencePath(), Utf8.toBytes(reference.value()))); + void writeApplicationPackageReference(FileReference applicationPackageReference) { + curator.set(applicationPackageReferencePath(), Utf8.toBytes(applicationPackageReference.value())); } FileReference readApplicationPackageReference() { Optional<byte[]> data = curator.getData(applicationPackageReferencePath()); - if (data.isEmpty()) return null; // This should not happen. - - return new FileReference(Utf8.toString(data.get())); + return new FileReference(Utf8.toString( + data.orElseThrow(() -> new IllegalArgumentException("No application package reference found")))); } private Path applicationPackageReferencePath() { @@ -228,9 +226,8 @@ public class SessionZooKeeperClient { } public Version readVespaVersion() { - Optional<byte[]> data = curator.getData(versionPath()); - // TODO: Empty version should not be possible any more - verify and remove - return data.map(d -> new Version(Utf8.toString(d))).orElse(Vtag.currentVersion); + return curator.getData(versionPath()).map(d -> new Version( + Utf8.toString(d))).orElseThrow(() -> new IllegalArgumentException("No vespa version found")); } public Optional<DockerImage> readDockerImageRepository() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java index 4a7aeafab7e..e5b44857685 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java @@ -134,7 +134,7 @@ public class SessionZooKeeperClientTest { public void require_that_application_package_file_reference_can_be_written_and_read() { final FileReference testRef = new FileReference("test-ref"); SessionZooKeeperClient zkc = createSessionZKClient(3); - zkc.writeApplicationPackageReference(Optional.of(testRef)); + zkc.writeApplicationPackageReference(testRef); assertEquals(testRef, zkc.readApplicationPackageReference()); } diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 757afeb64e2..6d7e3c86351 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -1027,6 +1027,45 @@ ], "fields" : [ ] }, + "com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Builder" : { + "superClass" : "java.lang.Object", + "interfaces" : [ + "com.yahoo.config.ConfigBuilder" + ], + "attributes" : [ + "public", + "final" + ], + "methods" : [ + "public void <init>()", + "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$AccessLog)", + "public com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Builder remoteAddressHeaders(java.lang.String)", + "public com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Builder remoteAddressHeaders(java.util.Collection)", + "public com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Builder remotePortHeaders(java.lang.String)", + "public com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Builder remotePortHeaders(java.util.Collection)", + "public com.yahoo.jdisc.http.ConnectorConfig$AccessLog build()" + ], + "fields" : [ + "public java.util.List remoteAddressHeaders", + "public java.util.List remotePortHeaders" + ] + }, + "com.yahoo.jdisc.http.ConnectorConfig$AccessLog" : { + "superClass" : "com.yahoo.config.InnerNode", + "interfaces" : [ ], + "attributes" : [ + "public", + "final" + ], + "methods" : [ + "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Builder)", + "public java.util.List remoteAddressHeaders()", + "public java.lang.String remoteAddressHeaders(int)", + "public java.util.List remotePortHeaders()", + "public java.lang.String remotePortHeaders(int)" + ], + "fields" : [ ] + }, "com.yahoo.jdisc.http.ConnectorConfig$Builder" : { "superClass" : "java.lang.Object", "interfaces" : [ @@ -1069,6 +1108,8 @@ "public com.yahoo.jdisc.http.ConnectorConfig$Builder http2(java.util.function.Consumer)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder serverName(com.yahoo.jdisc.http.ConnectorConfig$ServerName$Builder)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder serverName(java.util.function.Consumer)", + "public com.yahoo.jdisc.http.ConnectorConfig$Builder accessLog(com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Builder)", + "public com.yahoo.jdisc.http.ConnectorConfig$Builder accessLog(java.util.function.Consumer)", "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", "public final java.lang.String getDefMd5()", "public final java.lang.String getDefName()", @@ -1084,7 +1125,8 @@ "public com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy$Builder healthCheckProxy", "public com.yahoo.jdisc.http.ConnectorConfig$ProxyProtocol$Builder proxyProtocol", "public com.yahoo.jdisc.http.ConnectorConfig$Http2$Builder http2", - "public com.yahoo.jdisc.http.ConnectorConfig$ServerName$Builder serverName" + "public com.yahoo.jdisc.http.ConnectorConfig$ServerName$Builder serverName", + "public com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Builder accessLog" ] }, "com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy$Builder" : { @@ -1438,7 +1480,8 @@ "public double maxConnectionLife()", "public boolean http2Enabled()", "public com.yahoo.jdisc.http.ConnectorConfig$Http2 http2()", - "public com.yahoo.jdisc.http.ConnectorConfig$ServerName serverName()" + "public com.yahoo.jdisc.http.ConnectorConfig$ServerName serverName()", + "public com.yahoo.jdisc.http.ConnectorConfig$AccessLog accessLog()" ], "fields" : [ "public static final java.lang.String CONFIG_DEF_MD5", @@ -1771,45 +1814,6 @@ ], "fields" : [ ] }, - "com.yahoo.jdisc.http.ServerConfig$AccessLog$Builder" : { - "superClass" : "java.lang.Object", - "interfaces" : [ - "com.yahoo.config.ConfigBuilder" - ], - "attributes" : [ - "public", - "final" - ], - "methods" : [ - "public void <init>()", - "public void <init>(com.yahoo.jdisc.http.ServerConfig$AccessLog)", - "public com.yahoo.jdisc.http.ServerConfig$AccessLog$Builder remoteAddressHeaders(java.lang.String)", - "public com.yahoo.jdisc.http.ServerConfig$AccessLog$Builder remoteAddressHeaders(java.util.Collection)", - "public com.yahoo.jdisc.http.ServerConfig$AccessLog$Builder remotePortHeaders(java.lang.String)", - "public com.yahoo.jdisc.http.ServerConfig$AccessLog$Builder remotePortHeaders(java.util.Collection)", - "public com.yahoo.jdisc.http.ServerConfig$AccessLog build()" - ], - "fields" : [ - "public java.util.List remoteAddressHeaders", - "public java.util.List remotePortHeaders" - ] - }, - "com.yahoo.jdisc.http.ServerConfig$AccessLog" : { - "superClass" : "com.yahoo.config.InnerNode", - "interfaces" : [ ], - "attributes" : [ - "public", - "final" - ], - "methods" : [ - "public void <init>(com.yahoo.jdisc.http.ServerConfig$AccessLog$Builder)", - "public java.util.List remoteAddressHeaders()", - "public java.lang.String remoteAddressHeaders(int)", - "public java.util.List remotePortHeaders()", - "public java.lang.String remotePortHeaders(int)" - ], - "fields" : [ ] - }, "com.yahoo.jdisc.http.ServerConfig$Builder" : { "superClass" : "java.lang.Object", "interfaces" : [ @@ -1839,8 +1843,6 @@ "public com.yahoo.jdisc.http.ServerConfig$Builder jmx(java.util.function.Consumer)", "public com.yahoo.jdisc.http.ServerConfig$Builder metric(com.yahoo.jdisc.http.ServerConfig$Metric$Builder)", "public com.yahoo.jdisc.http.ServerConfig$Builder metric(java.util.function.Consumer)", - "public com.yahoo.jdisc.http.ServerConfig$Builder accessLog(com.yahoo.jdisc.http.ServerConfig$AccessLog$Builder)", - "public com.yahoo.jdisc.http.ServerConfig$Builder accessLog(java.util.function.Consumer)", "public com.yahoo.jdisc.http.ServerConfig$Builder connectionLog(com.yahoo.jdisc.http.ServerConfig$ConnectionLog$Builder)", "public com.yahoo.jdisc.http.ServerConfig$Builder connectionLog(java.util.function.Consumer)", "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", @@ -1856,7 +1858,6 @@ "public java.util.List defaultFilters", "public com.yahoo.jdisc.http.ServerConfig$Jmx$Builder jmx", "public com.yahoo.jdisc.http.ServerConfig$Metric$Builder metric", - "public com.yahoo.jdisc.http.ServerConfig$AccessLog$Builder accessLog", "public com.yahoo.jdisc.http.ServerConfig$ConnectionLog$Builder connectionLog" ] }, @@ -2070,7 +2071,6 @@ "public double stopTimeout()", "public com.yahoo.jdisc.http.ServerConfig$Jmx jmx()", "public com.yahoo.jdisc.http.ServerConfig$Metric metric()", - "public com.yahoo.jdisc.http.ServerConfig$AccessLog accessLog()", "public com.yahoo.jdisc.http.ServerConfig$ConnectionLog connectionLog()" ], "fields" : [ diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java index 5b51eeee7d6..7a305c23ba3 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java @@ -7,8 +7,6 @@ import com.yahoo.container.logging.AccessLogEntry; import com.yahoo.container.logging.RequestLog; import com.yahoo.container.logging.RequestLogEntry; import com.yahoo.jdisc.http.HttpRequest; -import com.yahoo.jdisc.http.ServerConfig; -import jakarta.servlet.http.HttpServletRequest; import org.eclipse.jetty.http2.HTTP2Stream; import org.eclipse.jetty.http2.server.HttpTransportOverHTTP2; import org.eclipse.jetty.server.HttpChannel; @@ -27,6 +25,7 @@ import java.util.function.BiConsumer; import java.util.logging.Level; import java.util.logging.Logger; +import static com.yahoo.jdisc.http.server.jetty.RequestUtils.getConnector; import static com.yahoo.jdisc.http.server.jetty.RequestUtils.getConnectorLocalPort; /** @@ -44,13 +43,9 @@ class AccessLogRequestLog extends AbstractLifeCycle implements org.eclipse.jetty private static final List<String> LOGGED_REQUEST_HEADERS = List.of("Vespa-Client-Version"); private final RequestLog requestLog; - private final List<String> remoteAddressHeaders; - private final List<String> remotePortHeaders; - AccessLogRequestLog(RequestLog requestLog, ServerConfig.AccessLog config) { + AccessLogRequestLog(RequestLog requestLog) { this.requestLog = requestLog; - this.remoteAddressHeaders = config.remoteAddressHeaders(); - this.remotePortHeaders = config.remotePortHeaders(); } @Override @@ -144,16 +139,16 @@ class AccessLogRequestLog extends AbstractLifeCycle implements org.eclipse.jetty } } - private String getRemoteAddress(HttpServletRequest request) { - for (String header : remoteAddressHeaders) { + private String getRemoteAddress(Request request) { + for (String header : getConnector(request).connectorConfig().accessLog().remoteAddressHeaders()) { String value = request.getHeader(header); if (value != null) return value; } return request.getRemoteAddr(); } - private int getRemotePort(HttpServletRequest request) { - for (String header : remotePortHeaders) { + private int getRemotePort(Request request) { + for (String header : getConnector(request).connectorConfig().accessLog().remotePortHeaders()) { String value = request.getHeader(header); if (value != null) { OptionalInt maybePort = parsePort(value); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java index 3ebb65e7979..7d84ee6f8a3 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java @@ -68,7 +68,7 @@ public class JettyHttpServer extends AbstractServerProvider { server = new Server(); server.setStopTimeout((long)(serverConfig.stopTimeout() * 1000.0)); - server.setRequestLog(new AccessLogRequestLog(requestLog, serverConfig.accessLog())); + server.setRequestLog(new AccessLogRequestLog(requestLog)); setupJmx(server, serverConfig); configureJettyThreadpool(server, serverConfig); JettyConnectionLogger connectionLogger = new JettyConnectionLogger(serverConfig.connectionLog(), connectionLog); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/CloudSslContextProvider.java b/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/CloudTokenSslContextProvider.java index cdfd4aa938e..fe71d1b24c6 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/CloudSslContextProvider.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/CloudTokenSslContextProvider.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.ssl.impl; +import com.yahoo.component.annotation.Inject; import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.jdisc.http.server.jetty.DataplaneProxyCredentials; @@ -14,29 +15,23 @@ import java.util.Optional; * * @author mortent */ -public class CloudSslContextProvider extends ConfiguredSslContextFactoryProvider { +public class CloudTokenSslContextProvider extends ConfiguredSslContextFactoryProvider { private final DataplaneProxyCredentials dataplaneProxyCredentials; - public CloudSslContextProvider(ConnectorConfig connectorConfig, DataplaneProxyCredentials dataplaneProxyCredentials) { + @Inject + public CloudTokenSslContextProvider(ConnectorConfig connectorConfig, + DataplaneProxyCredentials dataplaneProxyCredentials) { super(connectorConfig); this.dataplaneProxyCredentials = dataplaneProxyCredentials; } @Override Optional<String> getCaCertificates(ConnectorConfig.Ssl sslConfig) { - String proxyCert; try { - proxyCert = Files.readString(dataplaneProxyCredentials.certificateFile(), StandardCharsets.UTF_8); + return Optional.of(Files.readString(dataplaneProxyCredentials.certificateFile(), StandardCharsets.UTF_8)); } catch (IOException e) { throw new IllegalArgumentException("Dataplane proxy certificate not available", e); } - if (!sslConfig.caCertificate().isBlank()) { - return Optional.of(sslConfig.caCertificate() + "\n" + proxyCert); - } else if (!sslConfig.caCertificateFile().isBlank()) { - return Optional.of(readToString(sslConfig.caCertificateFile()) + "\n" + proxyCert); - } else { - return Optional.of(proxyCert); - } } } diff --git a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def index 3c01012fd9e..5a2bad63682 100644 --- a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def +++ b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def @@ -138,3 +138,9 @@ serverName.fallback string default="" # The list of accepted server names. Empty list to accept any. Elements follows format of 'serverName.default'. serverName.allowed[] string + +# HTTP request headers that contain remote address +accessLog.remoteAddressHeaders[] string + +# HTTP request headers that contain remote port +accessLog.remotePortHeaders[] string diff --git a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def index c15cb6b2cc4..a85641f61e9 100644 --- a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def +++ b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def @@ -52,11 +52,5 @@ metric.searchHandlerPaths[] string # User-agent names to ignore wrt statistics (crawlers etc) metric.ignoredUserAgents[] string -# HTTP request headers that contain remote address -accessLog.remoteAddressHeaders[] string - -# HTTP request headers that contain remote port -accessLog.remotePortHeaders[] string - # Whether to enable jdisc connection log connectionLog.enabled bool default=false diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLogTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLogTest.java index 766c7918882..122db0f765d 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLogTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLogTest.java @@ -4,7 +4,6 @@ package com.yahoo.jdisc.http.server.jetty; import com.yahoo.container.logging.AccessLogEntry; import com.yahoo.container.logging.RequestLog; import com.yahoo.container.logging.RequestLogEntry; -import com.yahoo.jdisc.http.ServerConfig; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Response; import org.junit.jupiter.api.Test; @@ -117,11 +116,7 @@ public class AccessLogRequestLogTest { } private void doAccessLoggingOfRequest(RequestLog requestLog, Request jettyRequest) { - ServerConfig.AccessLog config = new ServerConfig.AccessLog( - new ServerConfig.AccessLog.Builder() - .remoteAddressHeaders(List.of("x-forwarded-for", "y-ra")) - .remotePortHeaders(List.of("X-Forwarded-Port", "y-rp"))); - new AccessLogRequestLog(requestLog, config).log(jettyRequest, createResponseMock()); + new AccessLogRequestLog(requestLog).log(jettyRequest, createResponseMock()); } private static JettyMockRequestBuilder createRequestBuilder() { diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/JettyMockRequestBuilder.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/JettyMockRequestBuilder.java index e62825fc2a8..8b13f30bcd7 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/JettyMockRequestBuilder.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/JettyMockRequestBuilder.java @@ -85,7 +85,11 @@ public class JettyMockRequestBuilder { HttpChannel channel = mock(HttpChannel.class); HttpConnection connection = mock(HttpConnection.class); JDiscServerConnector connector = mock(JDiscServerConnector.class); - when(connector.connectorConfig()).thenReturn(new ConnectorConfig(new ConnectorConfig.Builder().listenPort(localPort))); + when(connector.connectorConfig()).thenReturn(new ConnectorConfig( + new ConnectorConfig.Builder().listenPort(localPort) + .accessLog(new ConnectorConfig.AccessLog.Builder() + .remoteAddressHeaders(List.of("x-forwarded-for", "y-ra")) + .remotePortHeaders(List.of("X-Forwarded-Port", "y-rp"))))); when(connector.getLocalPort()).thenReturn(localPort); when(connection.getCreatedTimeStamp()).thenReturn(System.currentTimeMillis()); when(connection.getConnector()).thenReturn(connector); diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java b/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java index 47050168b80..74e6954e1e1 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java @@ -103,7 +103,8 @@ public class DataplaneProxyService extends AbstractComponent { proxyCredentialsKey, serverCertificateFile, serverKeyFile, - config.port(), + config.mtlsPort(), + config.tokenPort(), root )); if (configChanged && state == NginxState.RUNNING) { @@ -191,7 +192,8 @@ public class DataplaneProxyService extends AbstractComponent { Path clientKey, Path serverCert, Path serverKey, - int vespaPort, + int vespaMtlsPort, + int vespaTokenPort, Path root) { try { @@ -200,7 +202,8 @@ public class DataplaneProxyService extends AbstractComponent { nginxTemplate = replace(nginxTemplate, "client_key", clientKey.toString()); nginxTemplate = replace(nginxTemplate, "server_cert", serverCert.toString()); nginxTemplate = replace(nginxTemplate, "server_key", serverKey.toString()); - nginxTemplate = replace(nginxTemplate, "vespa_port", Integer.toString(vespaPort)); + nginxTemplate = replace(nginxTemplate, "vespa_mtls_port", Integer.toString(vespaMtlsPort)); + nginxTemplate = replace(nginxTemplate, "vespa_token_port", Integer.toString(vespaTokenPort)); nginxTemplate = replace(nginxTemplate, "prefix", root.toString()); // TODO: verify that all template vars have been expanded diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/DataplaneProxyServiceTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/DataplaneProxyServiceTest.java index 351890e2a3a..893a527e631 100644 --- a/container-disc/src/test/java/com/yahoo/container/jdisc/DataplaneProxyServiceTest.java +++ b/container-disc/src/test/java/com/yahoo/container/jdisc/DataplaneProxyServiceTest.java @@ -168,7 +168,8 @@ public class DataplaneProxyServiceTest { private DataplaneProxyConfig proxyConfig() { X509CertificateWithKey selfSigned = X509CertificateUtils.createSelfSigned("cn=test", Duration.ofMinutes(10)); return new DataplaneProxyConfig.Builder() - .port(1234) + .mtlsPort(1234) + .tokenPort(1235) .serverCertificate(X509CertificateUtils.toPem(selfSigned.certificate())) .serverKey(KeyUtils.toPem(selfSigned.privateKey())) .build(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java index 485bf627c87..71003b9d86e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java @@ -55,7 +55,7 @@ public interface NodeRepository { void upgrade(ZoneId zone, NodeType type, Version version, boolean allowDowngrade); /** Upgrade OS for all nodes of given type to a new version */ - void upgradeOs(ZoneId zone, NodeType type, Version version); + void upgradeOs(ZoneId zone, NodeType type, Version version, boolean allowDowngrade); /** Get target versions for upgrades in given zone */ TargetVersions targetVersionsOf(ZoneId zone); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 54dcfa46188..bac2c0ab9d7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -126,7 +126,7 @@ import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toSet; /** - * A singleton owned by the Controller which contains the methods and state for controlling applications. + * A singleton owned by {@link Controller} which contains the methods and state for controlling applications. * * @author bratseth */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index f6bcbc9828b..6cbcc64cf33 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -30,9 +30,6 @@ import com.yahoo.vespa.hosted.controller.persistence.JobControlFlags; import com.yahoo.vespa.hosted.controller.restapi.dataplanetoken.DataplaneTokenService; import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.support.access.SupportAccessControl; -import com.yahoo.vespa.hosted.controller.versions.OsVersion; -import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; -import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; @@ -40,15 +37,12 @@ import com.yahoo.yolean.concurrent.Sleeper; import java.security.SecureRandom; import java.time.Clock; -import java.time.Instant; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Random; import java.util.Set; -import java.util.TreeSet; -import java.util.function.Function; import java.util.function.Predicate; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -85,6 +79,7 @@ public class Controller extends AbstractComponent { private final MavenRepository mavenRepository; private final Metric metric; private final RoutingController routingController; + private final OsController osController; private final ControllerConfig controllerConfig; private final SecretStore secretStore; private final CuratorArchiveBucketDb archiveBucketDb; @@ -132,6 +127,7 @@ public class Controller extends AbstractComponent { applicationController = new ApplicationController(this, curator, accessControl, clock, flagSource, serviceRegistry.billingController()); tenantController = new TenantController(this, curator, accessControl); routingController = new RoutingController(this, rotationsConfig); + osController = new OsController(this); auditLogger = new AuditLogger(curator, clock); jobControl = new JobControl(new JobControlFlags(curator, flagSource)); archiveBucketDb = new CuratorArchiveBucketDb(this); @@ -161,6 +157,11 @@ public class Controller extends AbstractComponent { return routingController; } + /** Returns the instance controlling OS upgrades */ + public OsController os() { + return osController; + } + /** Returns the service registry of this */ public ServiceRegistry serviceRegistry() { return serviceRegistry; @@ -232,80 +233,6 @@ public class Controller extends AbstractComponent { .orElse(Vtag.currentVersion); } - /** Returns the target OS version for infrastructure in this system. The controller will drive infrastructure OS - * upgrades to this version */ - public Optional<OsVersionTarget> osVersionTarget(CloudName cloud) { - return osVersionTargets().stream().filter(target -> target.osVersion().cloud().equals(cloud)).findFirst(); - } - - /** Returns all target OS versions in this system */ - public Set<OsVersionTarget> osVersionTargets() { - return curator.readOsVersionTargets(); - } - - /** Set the target OS version for given cloud in this system */ - public void upgradeOsIn(CloudName cloudName, Version version, boolean force) { - if (version.isEmpty()) { - throw new IllegalArgumentException("Invalid version '" + version.toFullString() + "'"); - } - if (!clouds().contains(cloudName)) { - throw new IllegalArgumentException("Cloud '" + cloudName + "' does not exist in this system"); - } - Instant scheduledAt = clock.instant(); - try (Mutex lock = curator.lockOsVersions()) { - Map<CloudName, OsVersionTarget> targets = curator.readOsVersionTargets().stream() - .collect(Collectors.toMap(t -> t.osVersion().cloud(), - Function.identity())); - - OsVersionTarget currentTarget = targets.get(cloudName); - if (!force && currentTarget != null) { - if (currentTarget.osVersion().version().isAfter(version)) { - throw new IllegalArgumentException("Cannot downgrade cloud '" + cloudName.value() + "' to version " + - version.toFullString()); - } - if (currentTarget.osVersion().version().equals(version)) return; // Version unchanged - } - - OsVersionTarget newTarget = new OsVersionTarget(new OsVersion(version, cloudName), scheduledAt); - targets.put(cloudName, newTarget); - curator.writeOsVersionTargets(new TreeSet<>(targets.values())); - log.info("Triggered OS upgrade to " + version.toFullString() + " in cloud " + cloudName.value()); - } - } - - /** Clear the target OS version for given cloud in this system */ - public void cancelOsUpgradeIn(CloudName cloudName) { - try (Mutex lock = curator.lockOsVersions()) { - Map<CloudName, OsVersionTarget> targets = curator.readOsVersionTargets().stream() - .collect(Collectors.toMap(t -> t.osVersion().cloud(), - Function.identity())); - if (targets.remove(cloudName) == null) { - throw new IllegalArgumentException("Cloud '" + cloudName.value() + " has no OS upgrade target"); - } - curator.writeOsVersionTargets(new TreeSet<>(targets.values())); - } - } - - /** Returns the current OS version status */ - public OsVersionStatus osVersionStatus() { - return curator.readOsVersionStatus(); - } - - /** Replace the current OS version status with a new one */ - public void updateOsVersionStatus(OsVersionStatus newStatus) { - try (Mutex lock = curator.lockOsVersionStatus()) { - OsVersionStatus currentStatus = curator.readOsVersionStatus(); - for (CloudName cloud : clouds()) { - Set<Version> newVersions = newStatus.versionsIn(cloud); - if (currentStatus.versionsIn(cloud).size() > 1 && newVersions.size() == 1) { - log.info("All nodes in " + cloud + " cloud upgraded to OS version " + - newVersions.iterator().next().toFullString()); - } - } - curator.writeOsVersionStatus(newStatus); - } - } - /** Returns the hostname of this controller */ public HostName hostname() { return serviceRegistry.getHostname(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/OsController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/OsController.java new file mode 100644 index 00000000000..9d480c57c7a --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/OsController.java @@ -0,0 +1,129 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.CloudName; +import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import com.yahoo.vespa.hosted.controller.versions.OsVersion; +import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; +import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; + +import java.time.Instant; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.function.Function; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * A singleton owned by {@link Controller} which contains the methods and state for controlling OS upgrades. + * + * @author mpolden + */ +public record OsController(Controller controller) { + + private static final Logger LOG = Logger.getLogger(OsController.class.getName()); + + public OsController { + Objects.requireNonNull(controller); + } + + /** Returns the target OS version for infrastructure in this system. The controller will drive infrastructure OS + * upgrades to this version */ + public Optional<OsVersionTarget> target(CloudName cloud) { + return targets().stream().filter(target -> target.osVersion().cloud().equals(cloud)).findFirst(); + } + + /** Returns all target OS versions in this system */ + public Set<OsVersionTarget> targets() { + return curator().readOsVersionTargets(); + } + + /** + * Set the target OS version for given cloud in this system. + * + * @param version The target OS version + * @param cloud The cloud to upgrade + * @param force Allow downgrades, and override pinned target (if any) + * @param pin Pin this version. This prevents automatic scheduling of upgrades until version is unpinned + */ + public void upgradeTo(Version version, CloudName cloud, boolean force, boolean pin) { + if (version.isEmpty()) { + throw new IllegalArgumentException("Invalid version '" + version.toFullString() + "'"); + } + if (!controller.clouds().contains(cloud)) { + throw new IllegalArgumentException("Cloud '" + cloud + "' does not exist in this system"); + } + Instant scheduledAt = controller.clock().instant(); + try (Mutex lock = curator().lockOsVersions()) { + Map<CloudName, OsVersionTarget> targets = curator().readOsVersionTargets().stream() + .collect(Collectors.toMap(t -> t.osVersion().cloud(), + Function.identity())); + + OsVersionTarget currentTarget = targets.get(cloud); + boolean downgrade = false; + if (currentTarget != null) { + boolean versionChange = !currentTarget.osVersion().version().equals(version); + downgrade = version.isBefore(currentTarget.osVersion().version()); + if (versionChange && currentTarget.pinned() && !force) { + throw new IllegalArgumentException("Cannot " + (downgrade ? "downgrade" : "upgrade") + " cloud " + + cloud.value() + "' to version " + version.toFullString() + + ": Current target is pinned. Add 'force' parameter to override"); + } + if (downgrade && !force) { + throw new IllegalArgumentException("Cannot downgrade cloud '" + cloud.value() + "' to version " + + version.toFullString() + ": Missing 'force' parameter"); + } + if (!versionChange && currentTarget.pinned() == pin) return; // No change + } + + OsVersionTarget newTarget = new OsVersionTarget(new OsVersion(version, cloud), scheduledAt, pin, downgrade); + targets.put(cloud, newTarget); + curator().writeOsVersionTargets(new TreeSet<>(targets.values())); + LOG.info("Triggered OS " + (downgrade ? "downgrade" : "upgrade") + " to " + version.toFullString() + + " in cloud " + cloud.value()); + } + } + + /** Clear the target OS version for given cloud in this system */ + public void cancelUpgrade(CloudName cloudName) { + try (Mutex lock = curator().lockOsVersions()) { + Map<CloudName, OsVersionTarget> targets = curator().readOsVersionTargets().stream() + .collect(Collectors.toMap(t -> t.osVersion().cloud(), + Function.identity())); + if (targets.remove(cloudName) == null) { + throw new IllegalArgumentException("Cloud '" + cloudName.value() + " has no OS upgrade target"); + } + curator().writeOsVersionTargets(new TreeSet<>(targets.values())); + } + } + + /** Returns the current OS version status */ + public OsVersionStatus status() { + return curator().readOsVersionStatus(); + } + + /** Replace the current OS version status with a new one */ + public void updateStatus(OsVersionStatus newStatus) { + try (Mutex lock = curator().lockOsVersionStatus()) { + OsVersionStatus currentStatus = curator().readOsVersionStatus(); + for (CloudName cloud : controller.clouds()) { + Set<Version> newVersions = newStatus.versionsIn(cloud); + if (currentStatus.versionsIn(cloud).size() > 1 && newVersions.size() == 1) { + LOG.info("All nodes in " + cloud + " cloud upgraded to OS version " + + newVersions.iterator().next().toFullString()); + } + } + curator().writeOsVersionStatus(newStatus); + } + } + + private CuratorDb curator() { + return controller.curator(); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index a635d512054..3d550973f22 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -66,8 +66,8 @@ import java.util.stream.Collectors; import static java.util.stream.Collectors.toMap; /** - * The routing controller encapsulates state and methods for inspecting and manipulating deployment endpoints in a - * hosted Vespa system. + * The routing controller is owned by {@link Controller} and encapsulates state and methods for inspecting and + * manipulating deployment endpoints in a hosted Vespa system. * * The one-stop shop for all your routing needs! * diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index b6a645d96d2..bf2f2ab90eb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -27,7 +27,7 @@ import java.util.logging.Level; import java.util.logging.Logger; /** - * A singleton owned by the Controller which contains the methods and state for controlling tenants. + * A singleton owned by the {@link Controller} which contains the methods and state for controlling tenants. * * @author bratseth * @author mpolden diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 67188eb5e3a..98de84216e0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -276,7 +276,7 @@ public class MetricsReporter extends ControllerMaintainer { } private Map<NodeVersion, Duration> osChangeDurations() { - return changeDurations(controller().osVersionStatus().versions().values(), Function.identity()); + return changeDurations(controller().os().status().versions().values(), Function.identity()); } private <V> Map<NodeVersion, Duration> changeDurations(Collection<V> versions, Function<V, List<NodeVersion>> versionsGetter) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java index f0d218ae6cf..ac76ecf4b2a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.OsRelease; import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; +import com.yahoo.yolean.Exceptions; import java.time.DayOfWeek; import java.time.Duration; @@ -19,6 +20,8 @@ import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; import java.util.Objects; import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; /** * Automatically schedule upgrades to the next OS version. @@ -27,6 +30,8 @@ import java.util.Optional; */ public class OsUpgradeScheduler extends ControllerMaintainer { + private static final Logger LOG = Logger.getLogger(OsUpgradeScheduler.class.getName()); + public OsUpgradeScheduler(Controller controller, Duration interval) { super(controller, interval); } @@ -34,18 +39,27 @@ public class OsUpgradeScheduler extends ControllerMaintainer { @Override protected double maintain() { Instant now = controller().clock().instant(); + int attempts = 0; + int failures = 0; for (var cloud : controller().clouds()) { Optional<Change> change = changeIn(cloud, now); if (change.isEmpty()) continue; if (!change.get().scheduleAt(now)) continue; - controller().upgradeOsIn(cloud, change.get().version(), false); + try { + attempts++; + controller().os().upgradeTo(change.get().version(), cloud, false, false); + } catch (IllegalArgumentException e) { + failures++; + LOG.log(Level.WARNING, "Failed to schedule OS upgrade: " + Exceptions.toMessageString(e) + + ". Retrying in " + interval()); + } } - return 0.0; + return asSuccessFactorDeviation(attempts, failures); } /** Returns the wanted change for cloud at given instant, if any */ public Optional<Change> changeIn(CloudName cloud, Instant instant) { - Optional<OsVersionTarget> currentTarget = controller().osVersionTarget(cloud); + Optional<OsVersionTarget> currentTarget = controller().os().target(cloud); if (currentTarget.isEmpty()) return Optional.empty(); if (upgradingToNewMajor(cloud)) return Optional.empty(); // Skip further upgrades until major version upgrade is complete @@ -54,7 +68,7 @@ public class OsUpgradeScheduler extends ControllerMaintainer { } private boolean upgradingToNewMajor(CloudName cloud) { - return controller().osVersionStatus().versionsIn(cloud).stream() + return controller().os().status().versionsIn(cloud).stream() .filter(version -> !version.isEmpty()) // Ignore empty/unknown versions .map(Version::getMajor) .distinct() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java index b44643b4405..4df40850cc9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java @@ -44,17 +44,18 @@ public class OsUpgrader extends InfrastructureUpgrader<OsVersionTarget> { @Override protected void upgrade(OsVersionTarget target, SystemApplication application, ZoneApi zone) { - log.info(Text.format("Upgrading OS of %s to version %s in %s in cloud %s", application.id(), - target.osVersion().version().toFullString(), - zone.getVirtualId(), zone.getCloudName())); + log.info(Text.format((target.downgrade() ? "Downgrading" : "Upgrading") + " OS of %s to version %s in %s in cloud %s", application.id(), + target.osVersion().version().toFullString(), + zone.getVirtualId(), zone.getCloudName())); controller().serviceRegistry().configServer().nodeRepository().upgradeOs(zone.getVirtualId(), application.nodeType(), - target.osVersion().version()); + target.osVersion().version(), + target.downgrade()); } @Override protected boolean convergedOn(OsVersionTarget target, SystemApplication application, ZoneApi zone, NodeSlice nodeSlice) { Version currentVersion = versionOf(nodeSlice, zone, application, Node::currentOsVersion).orElse(target.osVersion().version()); - return !currentVersion.isBefore(target.osVersion().version()); + return satisfiedBy(currentVersion, target); } @Override @@ -66,10 +67,10 @@ public class OsUpgrader extends InfrastructureUpgrader<OsVersionTarget> { @Override protected Optional<OsVersionTarget> target() { - // Return target if we have nodes in this cloud on a lower version - return controller().osVersionTarget(cloud) - .filter(target -> controller().osVersionStatus().nodesIn(cloud).stream() - .anyMatch(node -> node.currentVersion().isBefore(target.osVersion().version()))); + // Return target if we have nodes in this cloud on the wrong version + return controller().os().target(cloud) + .filter(target -> controller().os().status().nodesIn(cloud).stream() + .anyMatch(node -> !satisfiedBy(node.currentVersion(), target))); } @Override @@ -78,10 +79,19 @@ public class OsUpgrader extends InfrastructureUpgrader<OsVersionTarget> { return controller().serviceRegistry().configServer().nodeRepository() .targetVersionsOf(zone.getVirtualId()) .osVersion(application.nodeType()) - .map(currentTarget -> target.osVersion().version().isAfter(currentTarget)) + .map(currentVersion -> !satisfiedBy(currentVersion, target)) .orElse(true); } + private static boolean satisfiedBy(Version version, OsVersionTarget target) { + if (target.downgrade()) { + // When downgrading we want an exact version + return version.equals(target.osVersion().version()); + } + // Otherwise, matching or later version is fine + return !version.isBefore(target.osVersion().version()); + } + /** Returns whether node currently allows upgrades */ public static boolean canUpgrade(Node node, boolean includeDeferring) { return (includeDeferring || !node.deferOsUpgrade()) && upgradableNodeStates.contains(node.state()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java index c643df6af68..831b4275422 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java @@ -21,7 +21,7 @@ public class OsVersionStatusUpdater extends ControllerMaintainer { protected double maintain() { try { OsVersionStatus newStatus = OsVersionStatus.compute(controller()); - controller().updateOsVersionStatus(newStatus); + controller().os().updateStatus(newStatus); return 0.0; } catch (Exception e) { log.log(Level.WARNING, "Failed to compute OS version status: " + Exceptions.toMessageString(e) + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 89b4166e477..ae35306c783 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -63,6 +63,7 @@ import java.util.Map; import java.util.NavigableMap; import java.util.Optional; import java.util.Set; +import java.util.SortedSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; @@ -335,7 +336,7 @@ public class CuratorDb { // Infrastructure upgrades - public void writeOsVersionTargets(Set<OsVersionTarget> versions) { + public void writeOsVersionTargets(SortedSet<OsVersionTarget> versions) { curator.set(osVersionTargetsPath(), asJson(osVersionTargetSerializer.toSlime(versions))); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializer.java index c06a36d3a1d..a5e5d925865 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializer.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; import java.time.Instant; import java.util.Collections; import java.util.Set; +import java.util.SortedSet; import java.util.TreeSet; /** @@ -25,12 +26,14 @@ public class OsVersionTargetSerializer { private static final String versionsField = "versions"; private static final String scheduledAtField = "scheduledAt"; + private static final String pinnedField = "pinned"; + private static final String downgradeField = "downgrade"; public OsVersionTargetSerializer(OsVersionSerializer osVersionSerializer) { this.osVersionSerializer = osVersionSerializer; } - public Slime toSlime(Set<OsVersionTarget> osVersionTargets) { + public Slime toSlime(SortedSet<OsVersionTarget> osVersionTargets) { Slime slime = new Slime(); Cursor root = slime.setObject(); Cursor array = root.setArray(versionsField); @@ -44,7 +47,9 @@ public class OsVersionTargetSerializer { array.traverse((ArrayTraverser) (i, inspector) -> { OsVersion osVersion = osVersionSerializer.fromSlime(inspector); Instant scheduledAt = SlimeUtils.instant(inspector.field(scheduledAtField)); - osVersionTargets.add(new OsVersionTarget(osVersion, scheduledAt)); + boolean pinned = inspector.field(pinnedField).asBool(); + boolean downgrade = inspector.field(downgradeField).asBool(); + osVersionTargets.add(new OsVersionTarget(osVersion, scheduledAt, pinned, downgrade)); }); return Collections.unmodifiableSet(osVersionTargets); } @@ -52,6 +57,8 @@ public class OsVersionTargetSerializer { private void toSlime(OsVersionTarget target, Cursor object) { osVersionSerializer.toSlime(target.osVersion(), object); object.setLong(scheduledAtField, target.scheduledAt().toEpochMilli()); + object.setBool(pinnedField, target.pinned()); + object.setBool(downgradeField, target.downgrade()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java index cfd6ea3d6d9..0fa2dc492c2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java @@ -39,7 +39,6 @@ import java.util.Optional; import java.util.Set; import java.util.StringJoiner; import java.util.function.Function; -import java.util.stream.Collectors; /** * This implements the /os/v1 API which provides operators with information about, and scheduling of OS upgrades for @@ -142,24 +141,25 @@ public class OsApiHandler extends AuditLoggingRequestHandler { Inspector root = requestData.get(); CloudName cloud = parseStringField("cloud", root, CloudName::from); if (requireField("version", root).type() == Type.NIX) { - controller.cancelOsUpgradeIn(cloud); + controller.os().cancelUpgrade(cloud); return new MessageResponse("Cleared target OS version for cloud '" + cloud.value() + "'"); } Version target = parseStringField("version", root, Version::fromString); boolean force = root.field("force").asBool(); - controller.upgradeOsIn(cloud, target, force); + boolean pin = root.field("pin").asBool(); + controller.os().upgradeTo(target, cloud, force, pin); return new MessageResponse("Set target OS version for cloud '" + cloud.value() + "' to " + - target.toFullString()); + target.toFullString() + (pin ? " (pinned)" : "")); } private Slime osVersions() { Slime slime = new Slime(); Cursor root = slime.setObject(); - Set<OsVersionTarget> targets = controller.osVersionTargets(); + Set<OsVersionTarget> targets = controller.os().targets(); Cursor versions = root.setArray("versions"); Instant now = controller.clock().instant(); - controller.osVersionStatus().versions().forEach((osVersion, nodeVersions) -> { + controller.os().status().versions().forEach((osVersion, nodeVersions) -> { Cursor currentVersionObject = versions.addObject(); currentVersionObject.setString("version", osVersion.version().toFullString()); Optional<OsVersionTarget> target = targets.stream().filter(t -> t.osVersion().equals(osVersion)).findFirst(); @@ -167,6 +167,7 @@ public class OsApiHandler extends AuditLoggingRequestHandler { target.ifPresent(t -> { currentVersionObject.setString("upgradeBudget", Duration.ZERO.toString()); currentVersionObject.setLong("scheduledAt", t.scheduledAt().toEpochMilli()); + currentVersionObject.setBool("pinned", t.pinned()); Optional<Change> nextChange = osUpgradeScheduler.changeIn(t.osVersion().cloud(), now); nextChange.ifPresent(c -> { currentVersionObject.setString("nextVersion", c.version().toFullString()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index 6f9888b79e0..f90cee65058 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -14,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.maintenance.OsUpgrader; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -36,13 +37,15 @@ public record OsVersionStatus(Map<OsVersion, List<NodeVersion>> versions) { this.versions = ImmutableMap.copyOf(Objects.requireNonNull(versions, "versions must be non-null")); } - /** Returns nodes eligible for OS upgrades that exist in given cloud */ + /** Returns all node versions that exist in given cloud */ public List<NodeVersion> nodesIn(CloudName cloud) { - return versions.entrySet().stream() - .filter(entry -> entry.getKey().cloud().equals(cloud)) - .map(Map.Entry::getValue) - .findFirst() - .orElseGet(List::of); + List<NodeVersion> nodeVersions = new ArrayList<>(); + versions.forEach((osVersion, nodesOnVersion) -> { + if (osVersion.cloud().equals(cloud)) { + nodeVersions.addAll(nodesOnVersion); + } + }); + return Collections.unmodifiableList(nodeVersions); } /** Returns versions that exist in given cloud */ @@ -56,7 +59,7 @@ public record OsVersionStatus(Map<OsVersion, List<NodeVersion>> versions) { /** Compute the current OS versions in this system. This is expensive and should be called infrequently */ public static OsVersionStatus compute(Controller controller) { Map<OsVersion, List<NodeVersion>> osVersions = new HashMap<>(); - controller.osVersionTargets().forEach(target -> osVersions.put(target.osVersion(), new ArrayList<>())); + controller.os().targets().forEach(target -> osVersions.put(target.osVersion(), new ArrayList<>())); for (var application : SystemApplication.all()) { for (var zone : zonesToUpgrade(controller)) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java index 471670ec399..e9785216376 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java @@ -11,7 +11,7 @@ import java.util.Objects; * * @author mpolden */ -public record OsVersionTarget(OsVersion osVersion, Instant scheduledAt) implements VersionTarget, Comparable<OsVersionTarget> { +public record OsVersionTarget(OsVersion osVersion, Instant scheduledAt, boolean pinned, boolean downgrade) implements VersionTarget, Comparable<OsVersionTarget> { public OsVersionTarget { Objects.requireNonNull(osVersion); @@ -30,7 +30,7 @@ public record OsVersionTarget(OsVersion osVersion, Instant scheduledAt) implemen @Override public boolean downgrade() { - return false; // Not supported by this target type + return downgrade; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 7004028c072..5b70942bfd1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -169,7 +169,7 @@ public class NodeRepositoryMock implements NodeRepository { } @Override - public void upgradeOs(ZoneId zone, NodeType type, Version version) { + public void upgradeOs(ZoneId zone, NodeType type, Version version, boolean downgrade) { this.targetVersions.compute(zone, (ignored, targetVersions) -> { if (targetVersions == null) { targetVersions = TargetVersions.EMPTY; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 869a94c0449..53f2e85ad31 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -365,7 +365,7 @@ public class MetricsReporterTest { // All nodes upgrade to initial OS version var version0 = Version.fromString("8.0"); - tester.controller().upgradeOsIn(cloud, version0, false); + tester.controller().os().upgradeTo(version0, cloud, false, false); osUpgrader.maintain(); tester.configServer().setOsVersion(version0, SystemApplication.tenantHost.id(), zone); tester.configServer().setOsVersion(version0, SystemApplication.configServerHost.id(), zone); @@ -379,7 +379,7 @@ public class MetricsReporterTest { var currentVersion = i == 0 ? version0 : targets.get(i - 1); var nextVersion = targets.get(i); // System starts upgrading to next OS version - tester.controller().upgradeOsIn(cloud, nextVersion, false); + tester.controller().os().upgradeTo(nextVersion, cloud, false, false); runAll(osUpgrader, statusUpdater, reporter); assertOsChangeDuration(Duration.ZERO, hosts); assertOsNodeCount(hosts.size(), currentVersion); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java index 899745c7a39..27cdd847fb9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java @@ -43,49 +43,58 @@ public class OsUpgradeSchedulerTest { // Initial run does nothing as the cloud does not have a target scheduler.maintain(); - assertTrue(tester.controller().osVersionTarget(cloud).isEmpty(), "No target set"); + assertTrue(tester.controller().os().target(cloud).isEmpty(), "No target set"); // Target is set manually Version version0 = Version.fromString("7.0.0.20220101"); - tester.controller().upgradeOsIn(cloud, version0, false); + tester.controller().os().upgradeTo(version0, cloud, false, false); // Target remains unchanged as it hasn't expired yet for (var interval : List.of(Duration.ZERO, Duration.ofDays(30))) { tester.clock().advance(interval); scheduler.maintain(); - assertEquals(version0, tester.controller().osVersionTarget(cloud).get().osVersion().version()); + assertEquals(version0, tester.controller().os().target(cloud).get().osVersion().version()); } - // New release becomes available, but is not triggered until cool-down period has passed + // New release becomes available, but is not triggered until cool-down period has passed, and we're inside a + // trigger period Version version1 = Version.fromString("7.0.0.20220301"); tester.clock().advance(Duration.ofDays(16)); assertEquals("2022-03-03T09:05:00", formatInstant(tester.clock().instant())); assertEquals(version1, scheduler.changeIn(cloud, tester.clock().instant()).get().version()); scheduler.maintain(); assertEquals(version0, - tester.controller().osVersionTarget(cloud).get().osVersion().version(), + tester.controller().os().target(cloud).get().osVersion().version(), "Target is unchanged because cooldown hasn't passed"); - - // ... and we're inside the trigger period tester.clock().advance(Duration.ofDays(3).plusHours(18)); assertEquals("2022-03-07T03:05:00", formatInstant(tester.clock().instant())); scheduler.maintain(); assertEquals(version0, - tester.controller().osVersionTarget(cloud).get().osVersion().version(), + tester.controller().os().target(cloud).get().osVersion().version(), "Target is unchanged because we're outside trigger period"); tester.clock().advance(Duration.ofHours(5)); assertEquals("2022-03-07T08:05:00", formatInstant(tester.clock().instant())); + + // Time constraints have now passed, but the current target has been pinned in the meantime + tester.controller().os().upgradeTo(version0, cloud, false, true); Optional<OsUpgradeScheduler.Change> change = scheduler.changeIn(cloud, tester.clock().instant()); assertTrue(change.isPresent()); + assertEquals(-1, scheduler.maintain()); + assertEquals(version0, + tester.controller().os().target(cloud).get().osVersion().version(), + "Target is unchanged because it's pinned"); + + // Target is unpinned and new version is allowed to be scheduled + tester.controller().os().upgradeTo(version0, cloud, false, false); scheduler.maintain(); assertEquals(version1, - tester.controller().osVersionTarget(cloud).get().osVersion().version(), + tester.controller().os().target(cloud).get().osVersion().version(), "New target set"); // A few more days pass and target remains unchanged tester.clock().advance(Duration.ofDays(2)); scheduler.maintain(); - assertEquals(version1, tester.controller().osVersionTarget(cloud).get().osVersion().version()); + assertEquals(version1, tester.controller().os().target(cloud).get().osVersion().version()); // Estimate next change Optional<OsUpgradeScheduler.Change> nextChange = scheduler.changeIn(cloud, tester.clock().instant()); @@ -106,19 +115,19 @@ public class OsUpgradeSchedulerTest { // Set initial target Version version0 = Version.fromString("7.0.0.20220101"); - tester.controller().upgradeOsIn(cloud, version0, false); + tester.controller().os().upgradeTo(version0, cloud, false, false); // Next version is triggered Version version1 = Version.fromString("7.0.0.20220301"); tester.clock().advance(Duration.ofDays(44)); assertEquals("2022-03-01T02:05:00", formatInstant(tester.clock().instant())); scheduler.maintain(); - assertEquals(version0, tester.controller().osVersionTarget(cloud).get().osVersion().version()); + assertEquals(version0, tester.controller().os().target(cloud).get().osVersion().version()); // Cool-down passes tester.clock().advance(Duration.ofDays(1)); assertEquals(version1, scheduler.changeIn(cloud, tester.clock().instant()).get().version()); scheduler.maintain(); - assertEquals(version1, tester.controller().osVersionTarget(cloud).get().osVersion().version()); + assertEquals(version1, tester.controller().os().target(cloud).get().osVersion().version()); // Estimate next change Optional<OsUpgradeScheduler.Change> nextChange = scheduler.changeIn(cloud, tester.clock().instant()); @@ -137,7 +146,7 @@ public class OsUpgradeSchedulerTest { // Set initial target CloudName cloud = tester.controller().clouds().iterator().next(); Version version0 = Version.fromString("8.0"); - tester.controller().upgradeOsIn(cloud, version0, false); + tester.controller().os().upgradeTo(version0, cloud, false, false); // Stable release (tagged outside trigger period) is scheduled once trigger period opens Version version1 = Version.fromString("8.1"); @@ -151,7 +160,7 @@ public class OsUpgradeSchedulerTest { // A newer version is triggered manually Version version3 = Version.fromString("8.3"); - tester.controller().upgradeOsIn(cloud, version3, false); + tester.controller().os().upgradeTo(version3, cloud, false, false); // Nothing happens in next iteration as tagged release is older than manually triggered version scheduleUpgradeAfter(Duration.ofDays(7), version3, scheduler, tester); @@ -168,7 +177,7 @@ public class OsUpgradeSchedulerTest { // Set initial target CloudName cloud = tester.controller().clouds().iterator().next(); Version version0 = Version.fromString("8.0"); - tester.controller().upgradeOsIn(cloud, version0, false); + tester.controller().os().upgradeTo(version0, cloud, false, false); // Latest release is not scheduled immediately Version version1 = Version.fromString("8.1"); @@ -205,7 +214,7 @@ public class OsUpgradeSchedulerTest { tester.clock().advance(duration); scheduler.maintain(); CloudName cloud = tester.controller().clouds().iterator().next(); - OsVersionTarget target = tester.controller().osVersionTarget(cloud).get(); + OsVersionTarget target = tester.controller().os().target(cloud).get(); assertEquals(version, target.osVersion().version()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index adfb0286202..056db4f119c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -22,7 +22,6 @@ import java.util.Collection; import java.util.List; import java.util.function.Function; import java.util.function.UnaryOperator; -import java.util.stream.Collectors; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -72,9 +71,9 @@ public class OsUpgraderTest { // New OS version released Version version1 = Version.fromString("7.1"); - tester.controller().upgradeOsIn(cloud1, Version.fromString("7.0"), false); - tester.controller().upgradeOsIn(cloud1, version1, false); - assertEquals(1, tester.controller().osVersionTargets().size()); // Only allows one version per cloud + tester.controller().os().upgradeTo(Version.fromString("7.0"), cloud1, false, false); + tester.controller().os().upgradeTo(version1, cloud1, false, false); + assertEquals(1, tester.controller().os().targets().size()); // Only allows one version per cloud statusUpdater.maintain(); // zone 0: controllers upgrade first @@ -128,9 +127,9 @@ public class OsUpgraderTest { osUpgrader.maintain(); assertWanted(version1, SystemApplication.tenantHost, zone1, zone2, zone3, zone4); statusUpdater.maintain(); - assertTrue(tester.controller().osVersionStatus().nodesIn(cloud1).stream() - .filter(node -> !node.hostname().equals(nodeDeferringOsUpgrade.hostname())) - .allMatch(node -> node.currentVersion().equals(version1)), + assertTrue(tester.controller().os().status().nodesIn(cloud1).stream() + .filter(node -> !node.hostname().equals(nodeDeferringOsUpgrade.hostname())) + .allMatch(node -> node.currentVersion().equals(version1)), "All non-deferring nodes are on target version"); } @@ -151,8 +150,8 @@ public class OsUpgraderTest { // New OS version released Version version = Version.fromString("7.1"); - tester.controller().upgradeOsIn(cloud, Version.fromString("7.0"), false); - tester.controller().upgradeOsIn(cloud, version, false); // Replaces existing target + tester.controller().os().upgradeTo(Version.fromString("7.0"), cloud, false, false); + tester.controller().os().upgradeTo(version, cloud, false, false); // Replaces existing target statusUpdater.maintain(); // zone 1 upgrades @@ -173,12 +172,55 @@ public class OsUpgraderTest { // No more upgrades osUpgrader.maintain(); assertWanted(version, SystemApplication.tenantHost, zone1, zone2); - assertTrue(tester.controller().osVersionStatus().nodesIn(cloud).stream() - .noneMatch(node -> node.currentVersion().isBefore(version)), "All nodes on target version or newer"); + assertTrue(tester.controller().os().status().nodesIn(cloud).stream() + .noneMatch(node -> node.currentVersion().isBefore(version)), "All nodes on target version or newer"); + } + + @Test + public void downgrade_os() { + CloudName cloud = CloudName.from("cloud"); + ZoneApi zone1 = zone("dev.us-east-1", cloud); + ZoneApi zone2 = zone("prod.us-west-1", cloud); + UpgradePolicy upgradePolicy = UpgradePolicy.builder() + .upgrade(zone1) + .upgrade(zone2) + .build(); + OsUpgrader osUpgrader = osUpgrader(upgradePolicy, cloud, true); + + // Bootstrap system + tester.configServer().bootstrap(List.of(zone1.getId(), zone2.getId()), + List.of(SystemApplication.tenantHost)); + + // New OS version released + Version version0 = Version.fromString("1.0"); + Version version1 = Version.fromString("2.0"); + tester.controller().os().upgradeTo(version1, cloud, false, false); + statusUpdater.maintain(); + + // All zones upgrade + for (var zone : List.of(zone1, zone2)) { + osUpgrader.maintain(); + completeUpgrade(version1, SystemApplication.tenantHost, zone); + statusUpdater.maintain(); + } + assertTrue(tester.controller().os().status().nodesIn(cloud).stream() + .allMatch(node -> node.currentVersion().equals(version1)), "All nodes on target version"); + + // Downgrade is triggered + tester.controller().os().upgradeTo(version0, cloud, true, false); + + // All zones downgrade, in reverse order + for (var zone : List.of(zone2, zone1)) { + osUpgrader.maintain(); + completeUpgrade(version0, SystemApplication.tenantHost, zone); + statusUpdater.maintain(); + } + assertTrue(tester.controller().os().status().nodesIn(cloud).stream() + .allMatch(node -> node.currentVersion().equals(version0)), "All nodes on target version"); } private List<NodeVersion> nodesOn(Version version) { - return tester.controller().osVersionStatus().versions().entrySet().stream() + return tester.controller().os().status().versions().entrySet().stream() .filter(entry -> entry.getKey().version().equals(version)) .flatMap(entry -> entry.getValue().stream()) .toList(); @@ -241,22 +283,23 @@ public class OsUpgraderTest { completeUpgrade(nodeCount, version, version, application, zones); } - private void completeUpgrade(int nodeCount, Version wantedVersion, Version version, SystemApplication application, ZoneApi... zones) { + private void completeUpgrade(int nodeCount, Version wantedVersion, Version currentVersion, SystemApplication application, ZoneApi... zones) { assertWanted(wantedVersion, application, zones); for (ZoneApi zone : zones) { int nodesUpgraded = 0; List<Node> nodes = nodesRequiredToUpgrade(zone, application); for (Node node : nodes) { if (node.currentVersion().equals(wantedVersion)) continue; - nodeRepository().putNodes(zone.getVirtualId(), Node.builder(node).wantedOsVersion(version) - .currentOsVersion(version) + nodeRepository().putNodes(zone.getVirtualId(), Node.builder(node) + .wantedOsVersion(currentVersion) + .currentOsVersion(currentVersion) .build()); if (++nodesUpgraded == nodeCount) { break; } } if (nodesUpgraded == nodes.size()) { - assertCurrent(version, application, zone); + assertCurrent(currentVersion, application, zone); } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java index d4d146f0834..f45c7bfcdfb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java @@ -35,24 +35,24 @@ public class OsVersionStatusUpdaterTest { tester.zoneRegistry().setOsUpgradePolicy(CloudName.DEFAULT, upgradePolicy.build()); // Initially empty - assertSame(OsVersionStatus.empty, tester.controller().osVersionStatus()); + assertSame(OsVersionStatus.empty, tester.controller().os().status()); // Setting a new target adds it to current status Version version1 = Version.fromString("7.1"); CloudName cloud = CloudName.DEFAULT; - tester.controller().upgradeOsIn(cloud, version1, false); + tester.controller().os().upgradeTo(version1, cloud, false, false); statusUpdater.maintain(); - var osVersions = tester.controller().osVersionStatus().versions(); + var osVersions = tester.controller().os().status().versions(); assertEquals(3, osVersions.size()); assertFalse(osVersions.get(new OsVersion(Version.emptyVersion, cloud)).isEmpty(), "All nodes on unknown version"); assertTrue(osVersions.get(new OsVersion(version1, cloud)).isEmpty(), "No nodes on current target"); CloudName otherCloud = CloudName.AWS; - tester.controller().upgradeOsIn(otherCloud, version1, false); + tester.controller().os().upgradeTo(version1, otherCloud, false, false); statusUpdater.maintain(); - osVersions = tester.controller().osVersionStatus().versions(); + osVersions = tester.controller().os().status().versions(); assertEquals(4, osVersions.size()); // 2 in cloud, 2 in otherCloud. assertFalse(osVersions.get(new OsVersion(Version.emptyVersion, cloud)).isEmpty(), "All nodes on unknown version"); assertTrue(osVersions.get(new OsVersion(version1, cloud)).isEmpty(), "No nodes on current target"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializerTest.java index 55d940ca6f9..7bec217c889 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializerTest.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; -import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; import com.yahoo.vespa.hosted.controller.versions.OsVersion; @@ -10,6 +9,8 @@ import org.junit.jupiter.api.Test; import java.time.Instant; import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -21,10 +22,10 @@ public class OsVersionTargetSerializerTest { @Test void serialization() { OsVersionTargetSerializer serializer = new OsVersionTargetSerializer(new OsVersionSerializer()); - Set<OsVersionTarget> targets = ImmutableSet.of( - new OsVersionTarget(new OsVersion(Version.fromString("7.1"), CloudName.DEFAULT), Instant.ofEpochMilli(123)), - new OsVersionTarget(new OsVersion(Version.fromString("7.1"), CloudName.from("foo")), Instant.ofEpochMilli(456)) - ); + SortedSet<OsVersionTarget> targets = new TreeSet<>(); + targets.add(new OsVersionTarget(new OsVersion(Version.fromString("7.1"), CloudName.DEFAULT), Instant.ofEpochMilli(123), false, false)); + targets.add(new OsVersionTarget(new OsVersion(Version.fromString("7.1"), CloudName.from("foo")), Instant.ofEpochMilli(456), true, true)); + Set<OsVersionTarget> serialized = serializer.fromSlime(serializer.toSlime(targets)); assertEquals(targets, serialized); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java index 1067db31473..e569e0aca5b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java @@ -96,12 +96,20 @@ public class OsApiTest extends ControllerContainerTest { assertFile(new Request("http://localhost:8080/os/v1/"), "versions-all-upgraded.json"); // Downgrade with force is permitted - assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"7.5.1\", \"cloud\": \"cloud1\", \"force\": true}", Request.Method.PATCH), - "{\"message\":\"Set target OS version for cloud 'cloud1' to 7.5.1\"}", 200); + assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"8.2.0\", \"cloud\": \"cloud2\", \"force\": true}", Request.Method.PATCH), + "{\"message\":\"Set target OS version for cloud 'cloud2' to 8.2.0\"}", 200); // Clear target for a given cloud - assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": null, \"cloud\": \"cloud2\"}", Request.Method.PATCH), - "{\"message\":\"Cleared target OS version for cloud 'cloud2'\"}", 200); + assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": null, \"cloud\": \"cloud1\"}", Request.Method.PATCH), + "{\"message\":\"Cleared target OS version for cloud 'cloud1'\"}", 200); + + // Pin/unpin a version + assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"7.5.2\", \"cloud\": \"cloud1\", \"pin\": true}", Request.Method.PATCH), + "{\"message\":\"Set target OS version for cloud 'cloud1' to 7.5.2 (pinned)\"}", 200); + assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"7.5.2\", \"cloud\": \"cloud1\", \"pin\": false}", Request.Method.PATCH), + "{\"message\":\"Set target OS version for cloud 'cloud1' to 7.5.2\"}", 200); + assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"7.5.2\", \"cloud\": \"cloud1\", \"pin\": true}", Request.Method.PATCH), + "{\"message\":\"Set target OS version for cloud 'cloud1' to 7.5.2 (pinned)\"}", 200); // Error: Missing fields assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"7.6\"}", Request.Method.PATCH), @@ -120,8 +128,12 @@ public class OsApiTest extends ControllerContainerTest { "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cloud 'foo' does not exist in this system\"}", 400); // Error: Downgrade OS - assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"7.4.1\", \"cloud\": \"cloud1\"}", Request.Method.PATCH), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot downgrade cloud 'cloud1' to version 7.4.1\"}", 400); + assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"7.4.1\", \"cloud\": \"cloud2\"}", Request.Method.PATCH), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot downgrade cloud 'cloud2' to version 7.4.1: Missing 'force' parameter\"}", 400); + + // Error: Change a pinned version + assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"7.5.3\", \"cloud\": \"cloud1\"}", Request.Method.PATCH), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot upgrade cloud cloud1' to version 7.5.3: Current target is pinned. Add 'force' parameter to override\"}", 400); // Request firmware checks in all zones. assertResponse(new Request("http://localhost:8080/os/v1/firmware/", "", Request.Method.POST), @@ -148,7 +160,7 @@ public class OsApiTest extends ControllerContainerTest { } private void updateVersionStatus() { - tester.controller().updateOsVersionStatus(OsVersionStatus.compute(tester.controller())); + tester.controller().os().updateStatus(OsVersionStatus.compute(tester.controller())); } private void completeUpgrade(ZoneId... zones) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json index 92a2cad86f1..0f2e05986b6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json @@ -5,6 +5,7 @@ "targetVersion": true, "upgradeBudget": "PT0S", "scheduledAt": 1234, + "pinned": false, "cloud": "cloud1", "nodes": [ { @@ -104,6 +105,7 @@ "targetVersion": true, "upgradeBudget": "PT0S", "scheduledAt": 1234, + "pinned": false, "nextVersion": "8.2.1.20211228", "nextScheduledAt": 1640743200000, "cloud": "cloud2", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json index 662ff4bb373..20d7147a258 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json @@ -57,6 +57,7 @@ "targetVersion": true, "upgradeBudget": "PT0S", "scheduledAt": 1234, + "pinned": false, "cloud": "cloud1", "nodes": [ { @@ -163,6 +164,7 @@ "targetVersion": true, "upgradeBudget": "PT0S", "scheduledAt": 1234, + "pinned": false, "nextVersion": "8.2.1.20211228", "nextScheduledAt": 1640743200000, "cloud": "cloud2", diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp index e91e38e0fe4..1839005d720 100644 --- a/document/src/tests/serialization/vespadocumentserializer_test.cpp +++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp @@ -46,6 +46,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/document/base/exceptions.h> #include <vespa/vespalib/util/compressionconfig.h> +#include <filesystem> using vespalib::File; using vespalib::Slime; @@ -706,8 +707,8 @@ void checkDeserialization(const string &name, std::unique_ptr<Slime> slime) { PredicateFieldValue value(std::move(slime)); serializeToFile(value, data_dir + name + "__cpp.new"); - vespalib::rename(data_dir + name + "__cpp.new", - data_dir + name + "__cpp"); + std::filesystem::rename(std::filesystem::path(data_dir + name + "__cpp.new"), + std::filesystem::path(data_dir + name + "__cpp")); deserializeAndCheck(data_dir + name + "__cpp", value); deserializeAndCheck(data_dir + name + "__java", value); @@ -836,8 +837,8 @@ void checkDeserialization(const string &name, std::unique_ptr<vespalib::eval::Va value = std::move(tensor); } serializeToFile(value, data_dir + name + "__cpp.new"); - vespalib::rename(data_dir + name + "__cpp.new", - data_dir + name + "__cpp"); + std::filesystem::rename(std::filesystem::path(data_dir + name + "__cpp.new"), + std::filesystem::path(data_dir + name + "__cpp")); deserializeAndCheck(data_dir + name + "__cpp", value); deserializeAndCheck(data_dir + name + "__java", value); @@ -965,8 +966,8 @@ struct RefFixture { const string field_name = "ref_field"; serializeToFile(value, data_dir + file_base_name + "__cpp.new", fixed_repo.getDocumentTypeRepo(), ref_doc_type, field_name); - vespalib::rename(data_dir + file_base_name + "__cpp.new", - data_dir + file_base_name + "__cpp"); + std::filesystem::rename(std::filesystem::path(data_dir + file_base_name + "__cpp.new"), + std::filesystem::path(data_dir + file_base_name + "__cpp")); deserializeAndCheck(data_dir + file_base_name + "__cpp", value, fixed_repo, field_name); diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/ClientPrincipal.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/ClientPrincipal.java new file mode 100644 index 00000000000..bfb9bb920db --- /dev/null +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/ClientPrincipal.java @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.jdisc.http.filter.security.cloud; + +import com.yahoo.jdisc.http.filter.DiscFilterRequest; + +import java.security.Principal; +import java.util.Set; +import java.util.logging.Logger; + +/** + * @author bjorncs + */ +record ClientPrincipal(Set<String> ids, Set<Permission> permissions) implements Principal { + + private static final Logger log = Logger.getLogger(ClientPrincipal.class.getName()); + + ClientPrincipal { ids = Set.copyOf(ids); permissions = Set.copyOf(permissions); } + @Override public String getName() { + return "ids=%s,permissions=%s".formatted(ids, permissions.stream().map(Permission::asString).toList()); + } + + static void attachToRequest(DiscFilterRequest req, Set<String> ids, Set<Permission> permissions) { + var p = new ClientPrincipal(ids, permissions); + req.setUserPrincipal(p); + log.fine(() -> "Client with ids=%s, permissions=%s" + .formatted(ids, permissions.stream().map(Permission::asString).toList())); + } +} + diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudDataPlaneFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudDataPlaneFilter.java index 2dc80fc9d2b..379973cd8cf 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudDataPlaneFilter.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudDataPlaneFilter.java @@ -2,41 +2,25 @@ package com.yahoo.jdisc.http.filter.security.cloud; import com.yahoo.component.annotation.Inject; -import com.yahoo.component.provider.ComponentRegistry; -import com.yahoo.container.jdisc.AclMapping; -import com.yahoo.container.jdisc.RequestHandlerSpec; -import com.yahoo.container.jdisc.RequestView; -import com.yahoo.container.logging.AccessLogEntry; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; import com.yahoo.jdisc.http.filter.security.cloud.config.CloudDataPlaneFilterConfig; -import com.yahoo.jdisc.http.server.jetty.DataplaneProxyCredentials; import com.yahoo.security.X509CertificateUtils; -import com.yahoo.security.token.Token; -import com.yahoo.security.token.TokenCheckHash; -import com.yahoo.security.token.TokenDomain; -import com.yahoo.security.token.TokenFingerprint; -import java.security.Principal; import java.security.cert.X509Certificate; -import java.time.Clock; -import java.time.Instant; import java.util.ArrayList; import java.util.EnumSet; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.logging.Logger; -import java.util.stream.Collectors; -import static com.yahoo.jdisc.http.filter.security.cloud.CloudDataPlaneFilter.Permission.READ; -import static com.yahoo.jdisc.http.filter.security.cloud.CloudDataPlaneFilter.Permission.WRITE; -import static com.yahoo.jdisc.http.server.jetty.AccessLoggingRequestHandler.CONTEXT_KEY_ACCESS_LOG_ENTRY; +import static com.yahoo.jdisc.http.filter.security.cloud.Permission.READ; +import static com.yahoo.jdisc.http.filter.security.cloud.Permission.WRITE; + /** * Data plane filter for Cloud @@ -50,91 +34,49 @@ import static com.yahoo.jdisc.http.server.jetty.AccessLoggingRequestHandler.CONT public class CloudDataPlaneFilter extends JsonSecurityRequestFilterBase { private static final Logger log = Logger.getLogger(CloudDataPlaneFilter.class.getName()); - static final int CHECK_HASH_BYTES = 32; private final boolean legacyMode; private final List<Client> allowedClients; - private final TokenDomain tokenDomain; - private final Clock clock; @Inject - public CloudDataPlaneFilter(CloudDataPlaneFilterConfig cfg, - ComponentRegistry<DataplaneProxyCredentials> optionalReverseProxy) { - this(cfg, reverseProxyCert(optionalReverseProxy).orElse(null), Clock.systemUTC()); - } - - CloudDataPlaneFilter(CloudDataPlaneFilterConfig cfg, X509Certificate reverseProxyCert, Clock clock) { + public CloudDataPlaneFilter(CloudDataPlaneFilterConfig cfg) { this.legacyMode = cfg.legacyMode(); - this.tokenDomain = TokenDomain.of(cfg.tokenContext()); - this.clock = clock; if (legacyMode) { allowedClients = List.of(); log.fine(() -> "Legacy mode enabled"); } else { - allowedClients = parseClients(cfg, reverseProxyCert, clock); + allowedClients = parseClients(cfg); } } - private static Optional<X509Certificate> reverseProxyCert( - ComponentRegistry<DataplaneProxyCredentials> optionalReverseProxy) { - return optionalReverseProxy.allComponents().stream().findAny().map(DataplaneProxyCredentials::certificate); - } - - private static List<Client> parseClients(CloudDataPlaneFilterConfig cfg, X509Certificate reverseProxyCert, Clock clock) { - var now = clock.instant(); + private static List<Client> parseClients(CloudDataPlaneFilterConfig cfg) { Set<String> ids = new HashSet<>(); List<Client> clients = new ArrayList<>(cfg.clients().size()); - boolean hasClientRequiringCertificate = false; if (cfg.clients().isEmpty()) throw new IllegalArgumentException("Empty clients configuration"); for (var c : cfg.clients()) { if (ids.contains(c.id())) throw new IllegalArgumentException("Clients definition has duplicate id '%s'".formatted(c.id())); - if (!c.certificates().isEmpty() && !c.tokens().isEmpty()) - throw new IllegalArgumentException("Client '%s' has both certificate and token configured".formatted(c.id())); - if (c.certificates().isEmpty() && c.tokens().isEmpty()) - throw new IllegalArgumentException("Client '%s' has neither certificate nor token configured".formatted(c.id())); - if (!c.tokens().isEmpty() && reverseProxyCert == null) - throw new IllegalArgumentException( - "Client '%s' has token configured but reverse proxy certificate is missing".formatted(c.id())); + if (c.certificates().isEmpty()) + throw new IllegalArgumentException("Client '%s' has no certificate configured".formatted(c.id())); ids.add(c.id()); - EnumSet<Permission> permissions = c.permissions().stream().map(Permission::of) - .collect(Collectors.toCollection(() -> EnumSet.noneOf(Permission.class))); - if (!c.certificates().isEmpty()) { - List<X509Certificate> certs; - try { - certs = c.certificates().stream() - .flatMap(pem -> X509CertificateUtils.certificateListFromPem(pem).stream()).toList(); - } catch (Exception e) { - throw new IllegalArgumentException( - "Client '%s' contains invalid X.509 certificate PEM: %s".formatted(c.id(), e.toString()), e); - } - if (certs.isEmpty()) throw new IllegalArgumentException( - "Client '%s' certificate PEM contains no valid X.509 entries".formatted(c.id())); - clients.add(new Client(c.id(), permissions, certs, Map.of())); - hasClientRequiringCertificate = true; - } else { - var tokens = new HashMap<TokenCheckHash, TokenVersion>(); - for (var token : c.tokens()) { - for (int version = 0; version < token.checkAccessHashes().size(); version++) { - var tokenVersion = TokenVersion.of( - token.id(), token.fingerprints().get(version), token.checkAccessHashes().get(version), - token.expirations().get(version)); - tokens.put(tokenVersion.accessHash(), tokenVersion); - } - } - // Add reverse proxy certificate as required certificate for client definition - clients.add(new Client(c.id(), permissions, List.of(reverseProxyCert), tokens)); + List<X509Certificate> certs; + try { + certs = c.certificates().stream() + .flatMap(pem -> X509CertificateUtils.certificateListFromPem(pem).stream()).toList(); + } catch (Exception e) { + throw new IllegalArgumentException( + "Client '%s' contains invalid X.509 certificate PEM: %s".formatted(c.id(), e.toString()), e); } + if (certs.isEmpty()) throw new IllegalArgumentException( + "Client '%s' certificate PEM contains no valid X.509 entries".formatted(c.id())); + clients.add(new Client(c.id(), Permission.setOf(c.permissions()), certs)); } - if (!hasClientRequiringCertificate) - throw new IllegalArgumentException("At least one client must require a certificate"); log.fine(() -> "Configured clients with ids %s".formatted(ids)); return clients; } @Override protected Optional<ErrorResponse> filter(DiscFilterRequest req) { - var now = clock.instant(); var certs = req.getClientCertificateChain(); log.fine(() -> "Certificate chain contains %d elements".formatted(certs.size())); if (certs.isEmpty()) { @@ -143,109 +85,28 @@ public class CloudDataPlaneFilter extends JsonSecurityRequestFilterBase { } if (legacyMode) { log.fine("Legacy mode validation complete"); - req.setUserPrincipal(new ClientPrincipal(Set.of(), Set.of(READ, WRITE))); + ClientPrincipal.attachToRequest(req, Set.of(), Set.of(READ, WRITE)); return Optional.empty(); } - RequestView view = req.asRequestView(); - var permission = Optional.ofNullable((RequestHandlerSpec) req.getAttribute(RequestHandlerSpec.ATTRIBUTE_NAME)) - .or(() -> Optional.of(RequestHandlerSpec.DEFAULT_INSTANCE)) - .flatMap(spec -> { - var action = spec.aclMapping().get(view); - var maybePermission = Permission.of(action); - if (maybePermission.isEmpty()) log.fine(() -> "Unknown action '%s'".formatted(action)); - return maybePermission; - }).orElse(null); - if (permission == null) { - log.fine(() -> "No valid permission mapping defined for %s @ '%s'".formatted(view.method(), view.uri())); - return Optional.of(new ErrorResponse(Response.Status.FORBIDDEN, "Forbidden")); - } + var permission = Permission.getRequiredPermission(req).orElse(null); + if (permission == null) return Optional.of(new ErrorResponse(Response.Status.FORBIDDEN, "Forbidden")); var clientCert = certs.get(0); - var requestTokenHash = requestTokenHash(req).orElse(null); var clientIds = new TreeSet<String>(); var permissions = new TreeSet<Permission>(); - var matchedTokens = new HashSet<TokenVersion>(); for (Client c : allowedClients) { if (!c.permissions().contains(permission)) continue; if (!c.certificates().contains(clientCert)) continue; - if (!c.tokens().isEmpty()) { - if (requestTokenHash == null) continue; - var matchedToken = c.tokens().get(requestTokenHash); - if (matchedToken == null) continue; - var expiration = matchedToken.expiration().orElse(null); - if (expiration != null && now.isAfter(expiration)) continue; - matchedTokens.add(matchedToken); - } clientIds.add(c.id()); permissions.addAll(c.permissions()); } - if (matchedTokens.size() > 1) { - log.warning("Multiple tokens matched for request %s" - .formatted(matchedTokens.stream().map(TokenVersion::id).toList())); - return Optional.of(new ErrorResponse(Response.Status.FORBIDDEN, "Forbidden")); - } - var matchedToken = matchedTokens.stream().findAny().orElse(null); - if (matchedToken != null) { - addAccessLogEntry(req, "token.id", matchedToken.id()); - addAccessLogEntry(req, "token.hash", matchedToken.fingerprint().toDelimitedHexString()); - addAccessLogEntry(req, "token.exp", matchedToken.expiration().map(Instant::toString).orElse("<none>")); - } - log.fine(() -> "Client with ids=%s, permissions=%s" - .formatted(clientIds, permissions.stream().map(Permission::asString).toList())); if (clientIds.isEmpty()) return Optional.of(new ErrorResponse(Response.Status.FORBIDDEN, "Forbidden")); - req.setUserPrincipal(new ClientPrincipal(clientIds, permissions)); + ClientPrincipal.attachToRequest(req, clientIds, permissions); return Optional.empty(); } - private Optional<TokenCheckHash> requestTokenHash(DiscFilterRequest req) { - return Optional.ofNullable(req.getHeader("Authorization")) - .filter(h -> h.startsWith("Bearer ")) - .map(t -> t.substring("Bearer ".length()).trim()) - .map(t -> TokenCheckHash.of(Token.of(tokenDomain, t), CHECK_HASH_BYTES)); - } - - private static void addAccessLogEntry(DiscFilterRequest req, String key, String value) { - ((AccessLogEntry) req.getAttribute(CONTEXT_KEY_ACCESS_LOG_ENTRY)).addKeyValue(key, value); - } - - public record ClientPrincipal(Set<String> ids, Set<Permission> permissions) implements Principal { - public ClientPrincipal { ids = Set.copyOf(ids); permissions = Set.copyOf(permissions); } - @Override public String getName() { - return "ids=%s,permissions=%s".formatted(ids, permissions.stream().map(Permission::asString).toList()); - } - } - - enum Permission { READ, WRITE; - String asString() { - return switch (this) { - case READ -> "read"; - case WRITE -> "write"; - }; - } - static Permission of(String v) { - return switch (v) { - case "read" -> READ; - case "write" -> WRITE; - default -> throw new IllegalArgumentException("Invalid permission '%s'".formatted(v)); - }; - } - static Optional<Permission> of(AclMapping.Action a) { - if (a.equals(AclMapping.Action.READ)) return Optional.of(READ); - if (a.equals(AclMapping.Action.WRITE)) return Optional.of(WRITE); - return Optional.empty(); - } - } - - private record TokenVersion(String id, TokenFingerprint fingerprint, TokenCheckHash accessHash, Optional<Instant> expiration) { - static TokenVersion of(String id, String fingerprint, String accessHash, String expiration) { - return new TokenVersion(id, TokenFingerprint.ofHex(fingerprint), TokenCheckHash.ofHex(accessHash), - expiration.equals("<none>") ? Optional.empty() : Optional.of(Instant.parse(expiration))); - } - } - - private record Client(String id, EnumSet<Permission> permissions, List<X509Certificate> certificates, - Map<TokenCheckHash, TokenVersion> tokens) { + private record Client(String id, EnumSet<Permission> permissions, List<X509Certificate> certificates) { Client { - permissions = EnumSet.copyOf(permissions); certificates = List.copyOf(certificates); tokens = Map.copyOf(tokens); + permissions = EnumSet.copyOf(permissions); certificates = List.copyOf(certificates); } } } diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudTokenDataPlaneFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudTokenDataPlaneFilter.java new file mode 100644 index 00000000000..6597f10198d --- /dev/null +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudTokenDataPlaneFilter.java @@ -0,0 +1,146 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.filter.security.cloud; + +import com.yahoo.component.annotation.Inject; +import com.yahoo.container.logging.AccessLogEntry; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; +import com.yahoo.jdisc.http.filter.security.cloud.config.CloudTokenDataPlaneFilterConfig; +import com.yahoo.security.token.Token; +import com.yahoo.security.token.TokenCheckHash; +import com.yahoo.security.token.TokenDomain; +import com.yahoo.security.token.TokenFingerprint; + +import java.time.Clock; +import java.time.Instant; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.logging.Logger; + +import static com.yahoo.jdisc.http.server.jetty.AccessLoggingRequestHandler.CONTEXT_KEY_ACCESS_LOG_ENTRY; + +/** + * Token data plane filter for Cloud + * + * @author bjorncs + */ +public class CloudTokenDataPlaneFilter extends JsonSecurityRequestFilterBase { + + private static final Logger log = Logger.getLogger(CloudTokenDataPlaneFilter.class.getName()); + static final int CHECK_HASH_BYTES = 32; + + private final List<Client> allowedClients; + private final TokenDomain tokenDomain; + private final Clock clock; + + @Inject + public CloudTokenDataPlaneFilter(CloudTokenDataPlaneFilterConfig cfg) { + this(cfg, Clock.systemUTC()); + } + + CloudTokenDataPlaneFilter(CloudTokenDataPlaneFilterConfig cfg, Clock clock) { + this.tokenDomain = TokenDomain.of(cfg.tokenContext()); + this.clock = clock; + this.allowedClients = parseClients(cfg); + } + + private static List<Client> parseClients(CloudTokenDataPlaneFilterConfig cfg) { + Set<String> ids = new HashSet<>(); + List<Client> clients = new ArrayList<>(cfg.clients().size()); + if (cfg.clients().isEmpty()) throw new IllegalArgumentException("Empty clients configuration"); + for (var c : cfg.clients()) { + if (ids.contains(c.id())) + throw new IllegalArgumentException("Clients definition has duplicate id '%s'".formatted(c.id())); + if (c.tokens().isEmpty()) + throw new IllegalArgumentException("Client '%s' has no tokens configured".formatted(c.id())); + ids.add(c.id()); + var tokens = new HashMap<TokenCheckHash, TokenVersion>(); + for (var token : c.tokens()) { + for (int version = 0; version < token.checkAccessHashes().size(); version++) { + var tokenVersion = TokenVersion.of( + token.id(), token.fingerprints().get(version), token.checkAccessHashes().get(version), + token.expirations().get(version)); + tokens.put(tokenVersion.accessHash(), tokenVersion); + } + } + clients.add(new Client(c.id(), Permission.setOf(c.permissions()), tokens)); + } + log.fine(() -> "Configured clients with ids %s".formatted(ids)); + return List.copyOf(clients); + } + + @Override + protected Optional<ErrorResponse> filter(DiscFilterRequest req) { + var now = clock.instant(); + var bearerToken = requestBearerToken(req).orElse(null); + if (bearerToken == null) { + log.fine("Missing bearer token"); + return Optional.of(new ErrorResponse(Response.Status.UNAUTHORIZED, "Unauthorized")); + } + var permission = Permission.getRequiredPermission(req).orElse(null); + if (permission == null) return Optional.of(new ErrorResponse(Response.Status.FORBIDDEN, "Forbidden")); + var requestTokenHash = requestTokenHash(bearerToken); + var clientIds = new TreeSet<String>(); + var permissions = EnumSet.noneOf(Permission.class); + var matchedTokens = new HashSet<TokenVersion>(); + for (Client c : allowedClients) { + if (!c.permissions().contains(permission)) continue; + var matchedToken = c.tokens().get(requestTokenHash); + if (matchedToken == null) continue; + var expiration = matchedToken.expiration().orElse(null); + if (expiration != null && now.isAfter(expiration)) continue; + matchedTokens.add(matchedToken); + clientIds.add(c.id()); + permissions.addAll(c.permissions()); + } + if (clientIds.isEmpty()) return Optional.of(new ErrorResponse(Response.Status.FORBIDDEN, "Forbidden")); + if (matchedTokens.size() > 1) { + log.warning("Multiple tokens matched for request %s" + .formatted(matchedTokens.stream().map(TokenVersion::id).toList())); + return Optional.of(new ErrorResponse(Response.Status.FORBIDDEN, "Forbidden")); + } + var matchedToken = matchedTokens.stream().findAny().get(); + addAccessLogEntry(req, "token.id", matchedToken.id()); + addAccessLogEntry(req, "token.hash", matchedToken.fingerprint().toDelimitedHexString()); + addAccessLogEntry(req, "token.exp", matchedToken.expiration().map(Instant::toString).orElse("<none>")); + ClientPrincipal.attachToRequest(req, clientIds, permissions); + return Optional.empty(); + } + + private TokenCheckHash requestTokenHash(String bearerToken) { + return TokenCheckHash.of(Token.of(tokenDomain, bearerToken), CHECK_HASH_BYTES); + } + + private static Optional<String> requestBearerToken(DiscFilterRequest req) { + return Optional.ofNullable(req.getHeader("Authorization")) + .filter(h -> h.startsWith("Bearer ")) + .map(t -> t.substring("Bearer ".length()).trim()) + .filter(t -> !t.isBlank()); + + } + + private static void addAccessLogEntry(DiscFilterRequest req, String key, String value) { + ((AccessLogEntry) req.getAttribute(CONTEXT_KEY_ACCESS_LOG_ENTRY)).addKeyValue(key, value); + } + + private record TokenVersion(String id, TokenFingerprint fingerprint, TokenCheckHash accessHash, Optional<Instant> expiration) { + static TokenVersion of(String id, String fingerprint, String accessHash, String expiration) { + return new TokenVersion(id, TokenFingerprint.ofHex(fingerprint), TokenCheckHash.ofHex(accessHash), + expiration.equals("<none>") ? Optional.empty() : Optional.of(Instant.parse(expiration))); + } + } + + private record Client(String id, EnumSet<Permission> permissions, Map<TokenCheckHash, TokenVersion> tokens) { + Client { + permissions = EnumSet.copyOf(permissions); tokens = Map.copyOf(tokens); + } + } +} diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/Permission.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/Permission.java new file mode 100644 index 00000000000..4bab83f8576 --- /dev/null +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/Permission.java @@ -0,0 +1,63 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.jdisc.http.filter.security.cloud; + +import com.yahoo.container.jdisc.AclMapping; +import com.yahoo.container.jdisc.RequestHandlerSpec; +import com.yahoo.container.jdisc.RequestView; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; + +import java.util.Collection; +import java.util.EnumSet; +import java.util.Optional; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * @author bjorncs + */ +enum Permission { + READ, WRITE; + + private static final Logger log = Logger.getLogger(Permission.class.getName()); + + String asString() { + return switch (this) { + case READ -> "read"; + case WRITE -> "write"; + }; + } + + static Permission of(String v) { + return switch (v) { + case "read" -> READ; + case "write" -> WRITE; + default -> throw new IllegalArgumentException("Invalid permission '%s'".formatted(v)); + }; + } + + static EnumSet<Permission> setOf(Collection<String> v) { + return v.stream().map(Permission::of).collect(Collectors.toCollection(() -> EnumSet.noneOf(Permission.class))); + } + + static Optional<Permission> getRequiredPermission(DiscFilterRequest req) { + RequestView view = req.asRequestView(); + var result = Optional.ofNullable((RequestHandlerSpec) req.getAttribute(RequestHandlerSpec.ATTRIBUTE_NAME)) + .or(() -> Optional.of(RequestHandlerSpec.DEFAULT_INSTANCE)) + .flatMap(spec -> { + var action = spec.aclMapping().get(view); + var maybePermission = Permission.of(action); + if (maybePermission.isEmpty()) log.fine(() -> "Unknown action '%s'".formatted(action)); + return maybePermission; + }); + if (result.isEmpty()) + log.fine(() -> "No valid permission mapping defined for %s @ '%s'".formatted(view.method(), view.uri())); + return result; + } + + static Optional<Permission> of(AclMapping.Action a) { + if (a.equals(AclMapping.Action.READ)) return Optional.of(READ); + if (a.equals(AclMapping.Action.WRITE)) return Optional.of(WRITE); + return Optional.empty(); + } +} diff --git a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cloud/CloudDataPlaneFilterTest.java b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cloud/CloudDataPlaneFilterTest.java index d9daf8b6f46..8d2fd1f569e 100644 --- a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cloud/CloudDataPlaneFilterTest.java +++ b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cloud/CloudDataPlaneFilterTest.java @@ -5,35 +5,24 @@ import com.yahoo.container.jdisc.AclMapping.Action; import com.yahoo.container.jdisc.HttpMethodAclMapping; import com.yahoo.container.jdisc.RequestHandlerSpec; import com.yahoo.container.jdisc.RequestHandlerTestDriver.MockResponseHandler; -import com.yahoo.container.logging.AccessLogEntry; import com.yahoo.jdisc.http.HttpRequest.Method; -import com.yahoo.jdisc.http.filter.security.cloud.CloudDataPlaneFilter.ClientPrincipal; import com.yahoo.jdisc.http.filter.security.cloud.config.CloudDataPlaneFilterConfig; import com.yahoo.jdisc.http.filter.util.FilterTestUtils; import com.yahoo.security.KeyUtils; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.X509CertificateUtils; -import com.yahoo.security.token.Token; -import com.yahoo.security.token.TokenCheckHash; -import com.yahoo.security.token.TokenDomain; -import com.yahoo.security.token.TokenGenerator; -import com.yahoo.test.ManualClock; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import javax.security.auth.x500.X500Principal; import java.math.BigInteger; import java.security.cert.X509Certificate; -import java.time.Duration; -import java.time.Instant; import java.util.List; import java.util.Set; import static com.yahoo.jdisc.Response.Status.FORBIDDEN; import static com.yahoo.jdisc.Response.Status.UNAUTHORIZED; -import static com.yahoo.jdisc.http.filter.security.cloud.CloudDataPlaneFilter.CHECK_HASH_BYTES; -import static com.yahoo.jdisc.http.filter.security.cloud.CloudDataPlaneFilter.Permission.READ; -import static com.yahoo.jdisc.http.filter.security.cloud.CloudDataPlaneFilter.Permission.WRITE; +import static com.yahoo.jdisc.http.filter.security.cloud.Permission.READ; +import static com.yahoo.jdisc.http.filter.security.cloud.Permission.WRITE; import static com.yahoo.security.KeyAlgorithm.EC; import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; import static java.time.Instant.EPOCH; @@ -50,21 +39,8 @@ class CloudDataPlaneFilterTest { private static final X509Certificate FEED_CERT = certificate("my-feed-client"); private static final X509Certificate SEARCH_CERT = certificate("my-search-client"); private static final X509Certificate LEGACY_CLIENT_CERT = certificate("my-legacy-client"); - private static final X509Certificate REVERSE_PROXY_CERT = certificate("nginx"); private static final String FEED_CLIENT_ID = "feed-client"; private static final String MTLS_SEARCH_CLIENT_ID = "mtls-search-client"; - private static final String TOKEN_SEARCH_CLIENT = "token-search-client"; - private static final String TOKEN_CONTEXT = "my-token-context"; - private static final String TOKEN_ID = "my-token-id"; - private static final Instant TOKEN_EXPIRATION = EPOCH.plus(Duration.ofDays(1)); - private static final Token VALID_TOKEN = - TokenGenerator.generateToken(TokenDomain.of(TOKEN_CONTEXT), "vespa_token_", CHECK_HASH_BYTES); - private static final Token UNKNOWN_TOKEN = - TokenGenerator.generateToken(TokenDomain.of(TOKEN_CONTEXT), "vespa_token_", CHECK_HASH_BYTES); - - private ManualClock clock; - - @BeforeEach void resetClock() { clock = new ManualClock(EPOCH); } @Test void accepts_any_trusted_client_certificate_in_legacy_mode() { @@ -144,137 +120,13 @@ class CloudDataPlaneFilterTest { assertEquals(FORBIDDEN, responseHandler.getResponse().getStatus()); } - @Test - void accepts_reverse_proxy_with_token() { - var entry = new AccessLogEntry(); - var req = FilterTestUtils.newRequestBuilder() - .withMethod(Method.GET) - .withAccessLogEntry(entry) - .withClientCertificate(REVERSE_PROXY_CERT) - .withHeader("Authorization", "Bearer " + VALID_TOKEN.secretTokenString()) - .build(); - var responseHandler = new MockResponseHandler(); - newFilterWithClientsConfig().filter(req, responseHandler); - assertNull(responseHandler.getResponse()); - assertEquals(new ClientPrincipal(Set.of(TOKEN_SEARCH_CLIENT), Set.of(READ)), req.getUserPrincipal()); - assertEquals(TOKEN_ID, entry.getKeyValues().get("token.id").get(0)); - assertEquals(VALID_TOKEN.fingerprint().toDelimitedHexString(), entry.getKeyValues().get("token.hash").get(0)); - assertEquals(TOKEN_EXPIRATION.toString(), entry.getKeyValues().get("token.exp").get(0)); - } - - @Test - void fails_for_reverse_proxy_with_token_wrong_permission() { - var req = FilterTestUtils.newRequestBuilder() - .withMethod(Method.POST) - .withClientCertificate(REVERSE_PROXY_CERT) - .withHeader("Authorization", "Bearer " + VALID_TOKEN.secretTokenString()) - .build(); - var responseHandler = new MockResponseHandler(); - newFilterWithClientsConfig().filter(req, responseHandler); - assertNotNull(responseHandler.getResponse()); - assertEquals(FORBIDDEN, responseHandler.getResponse().getStatus()); - } - - @Test - void fails_for_reverse_proxy_without_token() { - var req = FilterTestUtils.newRequestBuilder() - .withMethod(Method.GET) - .withClientCertificate(REVERSE_PROXY_CERT) - .build(); - var responseHandler = new MockResponseHandler(); - newFilterWithClientsConfig().filter(req, responseHandler); - assertNotNull(responseHandler.getResponse()); - assertEquals(FORBIDDEN, responseHandler.getResponse().getStatus()); - } - - @Test - void fails_for_reverse_proxy_with_unknown_token() { - var req = FilterTestUtils.newRequestBuilder() - .withMethod(Method.GET) - .withClientCertificate(REVERSE_PROXY_CERT) - .withHeader("Authorization", "Bearer " + UNKNOWN_TOKEN.secretTokenString()) - .build(); - var responseHandler = new MockResponseHandler(); - newFilterWithClientsConfig().filter(req, responseHandler); - assertNotNull(responseHandler.getResponse()); - assertEquals(FORBIDDEN, responseHandler.getResponse().getStatus()); - } - - @Test - void fails_for_missing_certificate_with_token() { - var req = FilterTestUtils.newRequestBuilder() - .withMethod(Method.GET) - .withHeader("Authorization", "Bearer " + VALID_TOKEN.secretTokenString()) - .build(); - var responseHandler = new MockResponseHandler(); - newFilterWithClientsConfig().filter(req, responseHandler); - assertNotNull(responseHandler.getResponse()); - assertEquals(UNAUTHORIZED, responseHandler.getResponse().getStatus()); - } - - @Test - void fails_for_unknown_certificate_with_token() { - var req = FilterTestUtils.newRequestBuilder() - .withMethod(Method.GET) - .withClientCertificate(LEGACY_CLIENT_CERT) - .withHeader("Authorization", "Bearer " + VALID_TOKEN.secretTokenString()) - .build(); - var responseHandler = new MockResponseHandler(); - newFilterWithClientsConfig().filter(req, responseHandler); - assertNotNull(responseHandler.getResponse()); - assertEquals(FORBIDDEN, responseHandler.getResponse().getStatus()); - } - - @Test - void certificate_has_precedence_over_token() { - var req = FilterTestUtils.newRequestBuilder() - .withMethod(Method.POST) - .withClientCertificate(FEED_CERT) - .withHeader("Authorization", "Bearer " + VALID_TOKEN.secretTokenString()) - .build(); - var responseHandler = new MockResponseHandler(); - newFilterWithClientsConfig().filter(req, responseHandler); - assertNull(responseHandler.getResponse()); - assertEquals(new ClientPrincipal(Set.of(FEED_CLIENT_ID), Set.of(WRITE)), req.getUserPrincipal()); - } - - @Test - void fails_for_expired_token() { - var entry = new AccessLogEntry(); - var req = FilterTestUtils.newRequestBuilder() - .withMethod(Method.GET) - .withAccessLogEntry(entry) - .withClientCertificate(REVERSE_PROXY_CERT) - .withHeader("Authorization", "Bearer " + VALID_TOKEN.secretTokenString()) - .build(); - var filter = newFilterWithClientsConfig(); - - var responseHandler = new MockResponseHandler(); - filter.filter(req, responseHandler); - assertNull(responseHandler.getResponse()); - - clock.advance(Duration.ofDays(1)); - responseHandler = new MockResponseHandler(); - filter.filter(req, responseHandler); - assertNull(responseHandler.getResponse()); - - clock.advance(Duration.ofMillis(1)); - responseHandler = new MockResponseHandler(); - filter.filter(req, responseHandler); - assertNotNull(responseHandler.getResponse()); - assertEquals(FORBIDDEN, responseHandler.getResponse().getStatus()); - } - private CloudDataPlaneFilter newFilterWithLegacyMode() { - return new CloudDataPlaneFilter( - new CloudDataPlaneFilterConfig.Builder() - .legacyMode(true).build(), (X509Certificate) null, clock); + return new CloudDataPlaneFilter(new CloudDataPlaneFilterConfig.Builder().legacyMode(true).build()); } private CloudDataPlaneFilter newFilterWithClientsConfig() { return new CloudDataPlaneFilter( new CloudDataPlaneFilterConfig.Builder() - .tokenContext(TOKEN_CONTEXT) .clients(List.of( new CloudDataPlaneFilterConfig.Clients.Builder() .certificates(X509CertificateUtils.toPem(FEED_CERT)) @@ -283,18 +135,8 @@ class CloudDataPlaneFilterTest { new CloudDataPlaneFilterConfig.Clients.Builder() .certificates(X509CertificateUtils.toPem(SEARCH_CERT)) .permissions(READ.asString()) - .id(MTLS_SEARCH_CLIENT_ID), - new CloudDataPlaneFilterConfig.Clients.Builder() - .tokens(new CloudDataPlaneFilterConfig.Clients.Tokens.Builder() - .id(TOKEN_ID) - .checkAccessHashes(TokenCheckHash.of(VALID_TOKEN, 32).toHexString()) - .fingerprints(VALID_TOKEN.fingerprint().toDelimitedHexString()) - .expirations(TOKEN_EXPIRATION.toString())) - .permissions(READ.asString()) - .id(TOKEN_SEARCH_CLIENT))) - .build(), - REVERSE_PROXY_CERT, - clock); + .id(MTLS_SEARCH_CLIENT_ID))) + .build()); } private static X509Certificate certificate(String name) { diff --git a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cloud/CloudTokenDataPlaneFilterTest.java b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cloud/CloudTokenDataPlaneFilterTest.java new file mode 100644 index 00000000000..a34d2eb67c3 --- /dev/null +++ b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cloud/CloudTokenDataPlaneFilterTest.java @@ -0,0 +1,194 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.filter.security.cloud; + +import com.yahoo.container.jdisc.AclMapping.Action; +import com.yahoo.container.jdisc.HttpMethodAclMapping; +import com.yahoo.container.jdisc.RequestHandlerSpec; +import com.yahoo.container.jdisc.RequestHandlerTestDriver.MockResponseHandler; +import com.yahoo.container.logging.AccessLogEntry; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.jdisc.http.filter.security.cloud.config.CloudTokenDataPlaneFilterConfig; +import com.yahoo.jdisc.http.filter.util.FilterTestUtils; +import com.yahoo.security.token.Token; +import com.yahoo.security.token.TokenCheckHash; +import com.yahoo.security.token.TokenDomain; +import com.yahoo.security.token.TokenGenerator; +import com.yahoo.test.ManualClock; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Set; + +import static com.yahoo.jdisc.Response.Status.FORBIDDEN; +import static com.yahoo.jdisc.Response.Status.UNAUTHORIZED; +import static com.yahoo.jdisc.http.filter.security.cloud.CloudTokenDataPlaneFilter.CHECK_HASH_BYTES; +import static com.yahoo.jdisc.http.filter.security.cloud.Permission.READ; +import static com.yahoo.jdisc.http.filter.security.cloud.Permission.WRITE; +import static java.time.Instant.EPOCH; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +/** + * @author bjorncs + */ +class CloudTokenDataPlaneFilterTest { + + private static final String TOKEN_SEARCH_CLIENT = "token-search-client"; + private static final String TOKEN_FEED_CLIENT = "token-feed-client"; + private static final String TOKEN_CONTEXT = "my-token-context"; + private static final String READ_TOKEN_ID = "my-read-token-id"; + private static final String WRITE_TOKEN_ID = "my-write-token-id"; + private static final Instant TOKEN_EXPIRATION = EPOCH.plus(Duration.ofDays(1)); + private static final Token READ_TOKEN = + TokenGenerator.generateToken(TokenDomain.of(TOKEN_CONTEXT), "vespa_token_", CHECK_HASH_BYTES); + private static final Token WRITE_TOKEN = + TokenGenerator.generateToken(TokenDomain.of(TOKEN_CONTEXT), "vespa_token_", CHECK_HASH_BYTES); + private static final Token UNKNOWN_TOKEN = + TokenGenerator.generateToken(TokenDomain.of(TOKEN_CONTEXT), "vespa_token_", CHECK_HASH_BYTES); + private ManualClock clock; + + @BeforeEach void resetClock() { clock = new ManualClock(EPOCH); } + + @Test + void supports_handler_with_custom_request_spec() { + // Spec that maps POST as action 'read' + var spec = RequestHandlerSpec.builder() + .withAclMapping(HttpMethodAclMapping.standard() + .override(Method.POST, Action.READ).build()) + .build(); + var req = FilterTestUtils.newRequestBuilder() + .withMethod(Method.POST) + .withHeader("Authorization", "Bearer " + READ_TOKEN.secretTokenString()) + .withAttribute(RequestHandlerSpec.ATTRIBUTE_NAME, spec) + .build(); + var responseHandler = new MockResponseHandler(); + newFilterWithClientsConfig().filter(req, responseHandler); + assertNull(responseHandler.getResponse()); + assertEquals(new ClientPrincipal(Set.of(TOKEN_SEARCH_CLIENT), Set.of(READ)), req.getUserPrincipal()); + } + + @Test + void fails_on_handler_with_custom_request_spec_with_invalid_action() { + var spec = RequestHandlerSpec.builder() + .withAclMapping(HttpMethodAclMapping.standard() + .override(Method.GET, Action.custom("custom")).build()) + .build(); + var req = FilterTestUtils.newRequestBuilder() + .withMethod(Method.GET) + .withHeader("Authorization", "Bearer " + READ_TOKEN.secretTokenString()) + .withAttribute(RequestHandlerSpec.ATTRIBUTE_NAME, spec) + .build(); + var responseHandler = new MockResponseHandler(); + newFilterWithClientsConfig().filter(req, responseHandler); + assertNotNull(responseHandler.getResponse()); + assertEquals(FORBIDDEN, responseHandler.getResponse().getStatus()); + } + + @Test + void accepts_valid_token() { + var entry = new AccessLogEntry(); + var req = FilterTestUtils.newRequestBuilder() + .withMethod(Method.GET) + .withAccessLogEntry(entry) + .withHeader("Authorization", "Bearer " + READ_TOKEN.secretTokenString()) + .build(); + var responseHandler = new MockResponseHandler(); + newFilterWithClientsConfig().filter(req, responseHandler); + assertNull(responseHandler.getResponse()); + assertEquals(new ClientPrincipal(Set.of(TOKEN_SEARCH_CLIENT), Set.of(READ)), req.getUserPrincipal()); + assertEquals(READ_TOKEN_ID, entry.getKeyValues().get("token.id").get(0)); + assertEquals(READ_TOKEN.fingerprint().toDelimitedHexString(), entry.getKeyValues().get("token.hash").get(0)); + assertEquals(TOKEN_EXPIRATION.toString(), entry.getKeyValues().get("token.exp").get(0)); + } + + @Test + void fails_for_token_with_invalid_permission() { + var req = FilterTestUtils.newRequestBuilder() + .withMethod(Method.GET) + .withHeader("Authorization", "Bearer " + WRITE_TOKEN.secretTokenString()) + .build(); + var responseHandler = new MockResponseHandler(); + newFilterWithClientsConfig().filter(req, responseHandler); + assertNotNull(responseHandler.getResponse()); + assertEquals(FORBIDDEN, responseHandler.getResponse().getStatus()); + } + + @Test + void fails_for_missing_token() { + var req = FilterTestUtils.newRequestBuilder() + .withMethod(Method.GET) + .build(); + var responseHandler = new MockResponseHandler(); + newFilterWithClientsConfig().filter(req, responseHandler); + assertNotNull(responseHandler.getResponse()); + assertEquals(UNAUTHORIZED, responseHandler.getResponse().getStatus()); + } + + @Test + void fails_for_unknown_token() { + var req = FilterTestUtils.newRequestBuilder() + .withMethod(Method.GET) + .withHeader("Authorization", "Bearer " + UNKNOWN_TOKEN.secretTokenString()) + .build(); + var responseHandler = new MockResponseHandler(); + newFilterWithClientsConfig().filter(req, responseHandler); + assertNotNull(responseHandler.getResponse()); + assertEquals(FORBIDDEN, responseHandler.getResponse().getStatus()); + } + + @Test + void fails_for_expired_token() { + var entry = new AccessLogEntry(); + var req = FilterTestUtils.newRequestBuilder() + .withMethod(Method.GET) + .withAccessLogEntry(entry) + .withHeader("Authorization", "Bearer " + READ_TOKEN.secretTokenString()) + .build(); + var filter = newFilterWithClientsConfig(); + + var responseHandler = new MockResponseHandler(); + filter.filter(req, responseHandler); + assertNull(responseHandler.getResponse()); + + clock.advance(Duration.ofDays(1)); + responseHandler = new MockResponseHandler(); + filter.filter(req, responseHandler); + assertNull(responseHandler.getResponse()); + + clock.advance(Duration.ofMillis(1)); + responseHandler = new MockResponseHandler(); + filter.filter(req, responseHandler); + assertNotNull(responseHandler.getResponse()); + assertEquals(FORBIDDEN, responseHandler.getResponse().getStatus()); + } + + private CloudTokenDataPlaneFilter newFilterWithClientsConfig() { + return new CloudTokenDataPlaneFilter( + new CloudTokenDataPlaneFilterConfig.Builder() + .tokenContext(TOKEN_CONTEXT) + .clients(List.of( + new CloudTokenDataPlaneFilterConfig.Clients.Builder() + .tokens(new CloudTokenDataPlaneFilterConfig.Clients.Tokens.Builder() + .id(READ_TOKEN_ID) + .checkAccessHashes(TokenCheckHash.of(READ_TOKEN, 32).toHexString()) + .fingerprints(READ_TOKEN.fingerprint().toDelimitedHexString()) + .expirations(TOKEN_EXPIRATION.toString())) + .permissions(READ.asString()) + .id(TOKEN_SEARCH_CLIENT), + new CloudTokenDataPlaneFilterConfig.Clients.Builder() + .tokens(new CloudTokenDataPlaneFilterConfig.Clients.Tokens.Builder() + .id(WRITE_TOKEN_ID) + .checkAccessHashes(TokenCheckHash.of(WRITE_TOKEN, 32).toHexString()) + .fingerprints(WRITE_TOKEN.fingerprint().toDelimitedHexString()) + .expirations(TOKEN_EXPIRATION.toString())) + .permissions(WRITE.asString()) + .id(TOKEN_FEED_CLIENT))) + .build(), + clock); + } + +} diff --git a/screwdriver.yaml b/screwdriver.yaml index 19374a436d5..849f4a01328 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -426,10 +426,8 @@ jobs: fi publish-cli-release: + requires: [publish-release] image: homebrew/brew:latest - annotations: - # Run once an hour, in the hours 7-15 UTC, Monday-Thursday - screwdriver.cd/buildPeriodically: H 7-15 * * 1-4 secrets: - HOMEBREW_GITHUB_API_TOKEN - GH_TOKEN diff --git a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp index 47cbde152ef..85e610c092a 100644 --- a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp @@ -33,7 +33,6 @@ #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/transactionlog/translogserver.h> #include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/size_literals.h> @@ -333,11 +332,7 @@ TEST("require that resume after interrupted save config works") std::cout << "Best config serial is " << best_config_snapshot.syncToken << std::endl; auto old_config_subdir = config_subdir(best_config_snapshot.syncToken); auto new_config_subdir = config_subdir(serialNum + 1); - std::filesystem::create_directories(std::filesystem::path(new_config_subdir)); - auto config_files = vespalib::listDirectory(old_config_subdir); - for (auto &config_file : config_files) { - vespalib::copy(old_config_subdir + "/" + config_file, new_config_subdir + "/" + config_file, false, false); - } + std::filesystem::copy(std::filesystem::path(old_config_subdir), std::filesystem::path(new_config_subdir)); info.addSnapshot({true, serialNum + 1, new_config_subdir.substr(new_config_subdir.rfind('/') + 1)}); info.save(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp index 57a3d21652b..aecb1eec262 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp @@ -83,7 +83,9 @@ ProtonDiskLayout::remove(const DocTypeName &docTypeName) vespalib::string name(docTypeName.toString()); vespalib::string normalDir(documentsDir + "/" + name); vespalib::string removedDir(documentsDir + "/" + getRemovedName(name)); - vespalib::rename(normalDir, removedDir, false, false); + if (std::filesystem::exists(std::filesystem::path(normalDir))) { + std::filesystem::rename(std::filesystem::path(normalDir), std::filesystem::path(removedDir)); + } vespalib::File::sync(documentsDir); TransLogClient tlc(_transport, _tlsSpec); if (!tlc.remove(name)) { diff --git a/storage/src/vespa/storage/storageserver/statemanager.cpp b/storage/src/vespa/storage/storageserver/statemanager.cpp index 654fe0e1f5d..c228229e4ef 100644 --- a/storage/src/vespa/storage/storageserver/statemanager.cpp +++ b/storage/src/vespa/storage/storageserver/statemanager.cpp @@ -17,6 +17,7 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/string_escape.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/time.h> #include <fstream> #include <vespa/log/log.h> @@ -68,6 +69,10 @@ StateManager::StateManager(StorageComponentRegister& compReg, _threadLock(), _systemStateHistory(), _systemStateHistorySize(50), + _start_time(vespalib::steady_clock::now()), + _health_ping_time(), + _health_ping_warn_interval(5min), + _health_ping_warn_time(_start_time + _health_ping_warn_interval), _hostInfo(std::move(hostInfo)), _controllers_observed_explicit_node_state(), _noThreadTestMode(testMode), @@ -391,6 +396,8 @@ StateManager::onGetNodeState(const api::GetNodeStateCommand::SP& cmd) std::shared_ptr<api::GetNodeStateReply> reply; { std::unique_lock guard(_stateLock); + _health_ping_time = vespalib::steady_clock::now(); + _health_ping_warn_time = _health_ping_time.value() + _health_ping_warn_interval; const bool is_up_to_date = (_controllers_observed_explicit_node_state.find(cmd->getSourceIndex()) != _controllers_observed_explicit_node_state.end()); if ((cmd->getExpectedState() != nullptr) @@ -479,6 +486,28 @@ StateManager::run(framework::ThreadHandle& thread) } void +StateManager::warn_on_missing_health_ping() +{ + vespalib::steady_time now(vespalib::steady_clock::now()); + std::optional<vespalib::steady_time> health_ping_time; + { + std::lock_guard lock(_stateLock); + if (now <= _health_ping_warn_time) { + return; + } + health_ping_time = _health_ping_time; + _health_ping_warn_time = now + _health_ping_warn_interval; + } + if (health_ping_time.has_value()) { + vespalib::duration duration = now - health_ping_time.value(); + LOG(warning, "Last health ping from cluster controller was %1.1f seconds ago", vespalib::to_s(duration)); + } else { + vespalib::duration duration = now - _start_time; + LOG(warning, "No health pings from cluster controller since startup %1.1f seconds ago", vespalib::to_s(duration)); + } +} + +void StateManager::tick() { bool almost_immediate_replies = _requested_almost_immediate_node_state_replies.load(std::memory_order_relaxed); if (almost_immediate_replies) { @@ -487,6 +516,7 @@ StateManager::tick() { } else { sendGetNodeStateReplies(_component.getClock().getMonotonicTime()); } + warn_on_missing_health_ping(); } bool diff --git a/storage/src/vespa/storage/storageserver/statemanager.h b/storage/src/vespa/storage/storageserver/statemanager.h index 0b9a47c2515..3b1291b1c3f 100644 --- a/storage/src/vespa/storage/storageserver/statemanager.h +++ b/storage/src/vespa/storage/storageserver/statemanager.h @@ -65,6 +65,10 @@ class StateManager : public NodeStateUpdater, std::condition_variable _threadCond; std::deque<TimeSysStatePair> _systemStateHistory; uint32_t _systemStateHistorySize; + const vespalib::steady_time _start_time; + std::optional<vespalib::steady_time> _health_ping_time; + vespalib::duration _health_ping_warn_interval; + vespalib::steady_time _health_ping_warn_time; std::unique_ptr<HostInfo> _hostInfo; std::unique_ptr<framework::Thread> _thread; // Controllers that have observed a GetNodeState response sent _after_ @@ -84,6 +88,7 @@ public: void onClose() override; void tick(); + void warn_on_missing_health_ping(); void print(std::ostream& out, bool verbose, const std::string& indent) const override; void reportHtmlStatus(std::ostream&, const framework::HttpUrlPath&) const override; diff --git a/vespalib/src/tests/io/fileutil/fileutiltest.cpp b/vespalib/src/tests/io/fileutil/fileutiltest.cpp index 4eb700fd4ed..5a62e257032 100644 --- a/vespalib/src/tests/io/fileutil/fileutiltest.cpp +++ b/vespalib/src/tests/io/fileutil/fileutiltest.cpp @@ -235,133 +235,6 @@ TEST("require that vespalib::unlink works") } } -TEST("require that vespalib::rename works") -{ - std::filesystem::remove_all(std::filesystem::path("mydir")); - File f("myfile"); - f.open(File::CREATE | File::TRUNC); - f.write("Hello World!\n", 13, 0); - f.close(); - // Renaming to non-existing dir doesn't work - try{ - rename("myfile", "mydir/otherfile"); - TEST_FATAL("This shouldn't work when mydir doesn't exist"); - } catch (IoException& e) { - //std::cerr << e.what() << "\n"; - EXPECT_EQUAL(IoException::NOT_FOUND, e.getType()); - } - // Renaming to non-existing dir works if autocreating dirs - { - ASSERT_TRUE(rename("myfile", "mydir/otherfile", true, true)); - ASSERT_TRUE(!fileExists("myfile")); - ASSERT_TRUE(fileExists("mydir/otherfile")); - - File f2("mydir/otherfile"); - f2.open(File::READONLY); - std::vector<char> vec(20, ' '); - size_t read = f2.read(&vec[0], 20, 0); - EXPECT_EQUAL(13u, read); - EXPECT_EQUAL(std::string("Hello World!\n"), std::string(&vec[0], 13)); - } - // Renaming non-existing returns false - ASSERT_TRUE(!rename("myfile", "mydir/otherfile", true)); - // Rename to overwrite works - { - f.open(File::CREATE | File::TRUNC); - f.write("Bah\n", 4, 0); - f.close(); - ASSERT_TRUE(rename("myfile", "mydir/otherfile", true, true)); - - File f2("mydir/otherfile"); - f2.open(File::READONLY); - std::vector<char> vec(20, ' '); - size_t read = f2.read(&vec[0], 20, 0); - EXPECT_EQUAL(4u, read); - EXPECT_EQUAL(std::string("Bah\n"), std::string(&vec[0], 4)); - } - // Overwriting directory fails (does not put inside dir) - try{ - std::filesystem::create_directory(std::filesystem::path("mydir")); - f.open(File::CREATE | File::TRUNC); - f.write("Bah\n", 4, 0); - f.close(); - ASSERT_TRUE(rename("myfile", "mydir")); - } catch (IoException& e) { - //std::cerr << e.what() << "\n"; - EXPECT_EQUAL(IoException::ILLEGAL_PATH, e.getType()); - } - // Moving directory works - { - ASSERT_TRUE(isDirectory("mydir")); - std::filesystem::remove_all(std::filesystem::path("myotherdir")); - ASSERT_TRUE(rename("mydir", "myotherdir")); - ASSERT_TRUE(isDirectory("myotherdir")); - ASSERT_TRUE(!isDirectory("mydir")); - ASSERT_TRUE(!rename("mydir", "myotherdir")); - } - // Overwriting directory fails - try{ - File f2("mydir/yetanotherfile"); - f2.open(File::CREATE, true); - f2.write("foo", 3, 0); - f2.open(File::READONLY); - f2.close(); - rename("mydir", "myotherdir"); - TEST_FATAL("Should fail trying to overwrite directory"); - } catch (IoException& e) { - //std::cerr << e.what() << "\n"; - EXPECT_TRUE((IoException::DIRECTORY_HAVE_CONTENT == e.getType()) || - (IoException::ALREADY_EXISTS == e.getType())); - } -} - -TEST("require that vespalib::copy works") -{ - std::filesystem::remove_all(std::filesystem::path("mydir")); - File f("myfile"); - f.open(File::CREATE | File::TRUNC); - - MallocAutoPtr buffer = getAlignedBuffer(5000); - memset(buffer.get(), 0, 5000); - strncpy(static_cast<char*>(buffer.get()), "Hello World!\n", 14); - f.write(buffer.get(), 4_Ki, 0); - f.close(); - std::cerr << "Simple copy\n"; - // Simple copy works (4096b dividable file) - copy("myfile", "targetfile"); - ASSERT_TRUE(system("diff myfile targetfile") == 0); - std::cerr << "Overwriting\n"; - // Overwriting works (may not be able to use direct IO writing on all - // systems, so will always use cached IO) - { - f.open(File::CREATE | File::TRUNC); - f.write("Bah\n", 4, 0); - f.close(); - - ASSERT_TRUE(system("diff myfile targetfile > /dev/null") != 0); - copy("myfile", "targetfile"); - ASSERT_TRUE(system("diff myfile targetfile > /dev/null") == 0); - } - // Fails if target is directory - try{ - std::filesystem::create_directory(std::filesystem::path("mydir")); - copy("myfile", "mydir"); - TEST_FATAL("Should fail trying to overwrite directory"); - } catch (IoException& e) { - //std::cerr << e.what() << "\n"; - EXPECT_EQUAL(IoException::ILLEGAL_PATH, e.getType()); - } - // Fails if source is directory - try{ - std::filesystem::create_directory(std::filesystem::path("mydir")); - copy("mydir", "myfile"); - TEST_FATAL("Should fail trying to copy directory"); - } catch (IoException& e) { - //std::cerr << e.what() << "\n"; - EXPECT_EQUAL(IoException::ILLEGAL_PATH, e.getType()); - } -} - TEST("require that copy constructor and assignment for vespalib::File works") { // Copy file not opened. diff --git a/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp b/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp index 6662b2a4e41..62614f5d811 100644 --- a/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp +++ b/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp @@ -9,6 +9,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/testkit/time_bomb.h> #include <openssl/ssl.h> +#include <filesystem> using namespace vespalib; using namespace vespalib::net::tls; @@ -118,7 +119,7 @@ TEST_FF("Config reloading transitively loads updated files", Fixture(50ms), Time ASSERT_EQUAL(cert1_pem, current_certs); write_file("test_cert.pem.tmp", cert2_pem); - rename("test_cert.pem.tmp", "test_cert.pem", false, false); // We expect this to be an atomic rename under the hood + std::filesystem::rename(std::filesystem::path("test_cert.pem.tmp"), std::filesystem::path("test_cert.pem")); // We expect this to be an atomic rename under the hood current_certs = f1.current_cert_chain(); while (current_certs != cert2_pem) { @@ -140,7 +141,7 @@ TEST_FF("Config reload failure increments failure statistic", Fixture(50ms), Tim auto before = ConfigStatistics::get().snapshot(); write_file("test_cert.pem.tmp", "Broken file oh no :("); - rename("test_cert.pem.tmp", "test_cert.pem", false, false); + std::filesystem::rename(std::filesystem::path("test_cert.pem.tmp"), std::filesystem::path("test_cert.pem")); while (ConfigStatistics::get().snapshot().subtract(before).failed_config_reloads == 0) { std::this_thread::sleep_for(10ms); diff --git a/vespalib/src/vespa/vespalib/io/fileutil.cpp b/vespalib/src/vespa/vespalib/io/fileutil.cpp index ff39e56f000..491e07e2491 100644 --- a/vespalib/src/vespa/vespalib/io/fileutil.cpp +++ b/vespalib/src/vespa/vespalib/io/fileutil.cpp @@ -523,103 +523,12 @@ unlink(const string & filename) return true; } -bool -rename(const string & frompath, const string & topath, - bool copyDeleteBetweenFilesystems, bool createTargetDirectoryIfMissing) -{ - LOG(spam, "rename(%s, %s): Renaming file%s.", - frompath.c_str(), topath.c_str(), - createTargetDirectoryIfMissing - ? " recursively creating target directory if missing" : ""); - if (::rename(frompath.c_str(), topath.c_str()) != 0) { - if (errno == ENOENT) { - if (!fileExists(frompath)) return false; - if (createTargetDirectoryIfMissing) { - string::size_type pos = topath.rfind('/'); - if (pos != string::npos) { - string path(topath.substr(0, pos)); - std::filesystem::create_directories(std::filesystem::path(path)); - LOG(debug, "rename(%s, %s): Created target directory. Calling recursively.", - frompath.c_str(), topath.c_str()); - return rename(frompath, topath, copyDeleteBetweenFilesystems, false); - } - } else { - asciistream ost; - ost << "rename(" << frompath << ", " << topath - << (copyDeleteBetweenFilesystems ? ", revert to copy" : "") - << (createTargetDirectoryIfMissing - ? ", create missing target" : "") - << "): Failed, target path does not exist."; - throw IoException(ost.str(), IoException::NOT_FOUND, - VESPA_STRLOC); - } - } else if (errno == EXDEV && copyDeleteBetweenFilesystems) { - if (!fileExists(frompath)) { - LOG(debug, "rename(%s, %s): Renaming non-existing file across " - "filesystems returned EXDEV rather than ENOENT.", - frompath.c_str(), topath.c_str()); - return false; - } - LOG(debug, "rename(%s, %s): Cannot rename across filesystems. " - "Copying and deleting instead.", - frompath.c_str(), topath.c_str()); - copy(frompath, topath, createTargetDirectoryIfMissing); - unlink(frompath); - return true; - } - asciistream ost; - ost << "rename(" << frompath << ", " << topath - << (copyDeleteBetweenFilesystems ? ", revert to copy" : "") - << (createTargetDirectoryIfMissing ? ", create missing target" : "") - << "): Failed, errno(" << errno << "): " << safeStrerror(errno); - throw IoException(ost.str(), IoException::getErrorType(errno), - VESPA_STRLOC); - } - LOG(debug, "rename(%s, %s): Renamed.", frompath.c_str(), topath.c_str()); - return true; -} - namespace { - uint32_t bufferSize = 1_Mi; uint32_t diskAlignmentSize = 4_Ki; } -void -copy(const string & frompath, const string & topath, - bool createTargetDirectoryIfMissing, bool useDirectIO) -{ - // Get aligned buffer, so it works with direct IO - LOG(spam, "copy(%s, %s): Copying file%s.", - frompath.c_str(), topath.c_str(), - createTargetDirectoryIfMissing - ? " recursively creating target directory if missing" : ""); - MallocAutoPtr buffer(getAlignedBuffer(bufferSize)); - - File source(frompath); - File target(topath); - source.open(File::READONLY | (useDirectIO ? File::DIRECTIO : 0)); - size_t sourceSize = source.getFileSize(); - if (useDirectIO && sourceSize % diskAlignmentSize != 0) { - LOG(warning, "copy(%s, %s): Cannot use direct IO to write new file, " - "as source file has size %zu, which is not " - "dividable by the disk alignment size of %u.", - frompath.c_str(), topath.c_str(), sourceSize, diskAlignmentSize); - useDirectIO = false; - } - target.open(File::CREATE | File::TRUNC | (useDirectIO ? File::DIRECTIO : 0), - createTargetDirectoryIfMissing); - off_t offset = 0; - for (;;) { - size_t bytesRead = source.read(buffer.get(), bufferSize, offset); - target.write(buffer.get(), bytesRead, offset); - if (bytesRead < bufferSize) break; - offset += bytesRead; - } - LOG(debug, "copy(%s, %s): Completed.", frompath.c_str(), topath.c_str()); -} - DirectoryList listDirectory(const string & path) { diff --git a/vespalib/src/vespa/vespalib/io/fileutil.h b/vespalib/src/vespa/vespalib/io/fileutil.h index 7d9e51532d0..faffc739720 100644 --- a/vespalib/src/vespa/vespalib/io/fileutil.h +++ b/vespalib/src/vespa/vespalib/io/fileutil.h @@ -334,36 +334,6 @@ extern vespalib::string readLink(const vespalib::string & path); extern bool unlink(const vespalib::string & filename); /** - * Rename the file at frompath to topath. - * - * @param frompath old name of file. - * @param topath new name of file. - * - * @param copyDeleteBetweenFilesystems whether a copy-and-delete - * operation should be performed if rename crosses a file system - * boundary, or not. - * - * @param createTargetDirectoryIfMissing whether the target directory - * should be created if it's missing, or not. - * - * @throw IoException If we failed to rename the file. - * @throw std::filesystem::filesystem_error If we failed to create a target directory - * @return True if file was renamed, false if frompath did not exist. - */ -extern bool rename(const vespalib::string & frompath, - const vespalib::string & topath, - bool copyDeleteBetweenFilesystems = true, - bool createTargetDirectoryIfMissing = false); - -/** - * Copies a file to a destination using Direct IO. - */ -extern void copy(const vespalib::string & frompath, - const vespalib::string & topath, - bool createTargetDirectoryIfMissing = false, - bool useDirectIO = true); - -/** * List the contents of the given directory. */ using DirectoryList = std::vector<vespalib::string>; |