diff options
| author | Harald Musum <musum@yahooinc.com> | 2022-11-28 12:56:45 +0100 |
|---|---|---|
| committer | Harald Musum <musum@yahooinc.com> | 2022-11-28 12:56:45 +0100 |
| commit | dabbaa598b98c5e76f896a578de13eeefadee2de (patch) | |
| tree | 41f7b6aef4a28a62f52e2f8d952bdad0634bdd9e | |
| parent | fae673495fe1fd1c40d3704b390f52ef50ff856a (diff) | |
| parent | 5b45d91ced001b47ab253841e40d8ae3df8fec2b (diff) | |
Merge branch 'master' into hmusum/upgrade-curator-2
68 files changed, 1036 insertions, 110 deletions
diff --git a/application-model/pom.xml b/application-model/pom.xml index 2143f3a5ffd..c47fccc51bc 100644 --- a/application-model/pom.xml +++ b/application-model/pom.xml @@ -22,6 +22,10 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-compress</artifactId> + </dependency> + <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> @@ -39,6 +43,11 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> diff --git a/vespajlib/src/main/java/com/yahoo/compress/ArchiveStreamReader.java b/application-model/src/main/java/com/yahoo/vespa/archive/ArchiveStreamReader.java index f8faf655415..87665efc1ef 100644 --- a/vespajlib/src/main/java/com/yahoo/compress/ArchiveStreamReader.java +++ b/application-model/src/main/java/com/yahoo/vespa/archive/ArchiveStreamReader.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.compress; +package com.yahoo.vespa.archive; import com.yahoo.path.Path; import com.yahoo.yolean.Exceptions; diff --git a/application-model/src/main/java/com/yahoo/vespa/archive/package-info.java b/application-model/src/main/java/com/yahoo/vespa/archive/package-info.java new file mode 100644 index 00000000000..51b9d930ae6 --- /dev/null +++ b/application-model/src/main/java/com/yahoo/vespa/archive/package-info.java @@ -0,0 +1,5 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.archive; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/test/java/com/yahoo/compress/ArchiveStreamReaderTest.java b/application-model/src/test/java/com/yahoo/vespa/archive/ArchiveStreamReaderTest.java index b7f019282b7..78ff2a805e5 100644 --- a/vespajlib/src/test/java/com/yahoo/compress/ArchiveStreamReaderTest.java +++ b/application-model/src/test/java/com/yahoo/vespa/archive/ArchiveStreamReaderTest.java @@ -1,6 +1,6 @@ -package com.yahoo.compress; +package com.yahoo.vespa.archive; -import com.yahoo.compress.ArchiveStreamReader.Options; +import com.yahoo.vespa.archive.ArchiveStreamReader.Options; import com.yahoo.yolean.Exceptions; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 5e439a870f6..3fce00c3b16 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -131,6 +131,7 @@ <include>com.yahoo.vespa:jdisc_core:*:provided</include> <include>com.yahoo.vespa:jrt:*:provided</include> <include>com.yahoo.vespa:linguistics:*:provided</include> + <include>com.yahoo.vespa:opennlp-linguistics:*:provided</include> <include>com.yahoo.vespa:messagebus:*:provided</include> <include>com.yahoo.vespa:model-evaluation:*:provided</include> <include>com.yahoo.vespa:predicate-search-core:*:provided</include> @@ -186,7 +187,6 @@ <include>org.antlr:antlr-runtime:3.5.2:test</include> <include>org.antlr:antlr4-runtime:4.9.3:test</include> <include>org.apache.commons:commons-exec:1.3:test</include> - <include>org.apache.commons:commons-compress:1.21:test</include> <include>org.apache.commons:commons-math3:3.6.1:test</include> <include>org.apache.felix:org.apache.felix.framework:${felix.version}:test</include> <include>org.apache.felix:org.apache.felix.log:1.0.1:test</include> diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 918e91ed0ce..a4e62dc5488 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -111,6 +111,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"vekterli"}) default boolean useTwoPhaseDocumentGc() { return false; } @ModelFeatureFlag(owners = {"tokle"}) default boolean useRestrictedDataPlaneBindings() { return false; } @ModelFeatureFlag(owners = {"arnej","baldersheim"}) default boolean useOldJdiscContainerStartup() { return true; } + @ModelFeatureFlag(owners = {"tokle, bjorncs"}) default boolean enableDataPlaneFilter() { return false; } //Below are all flags that must be kept until 7 is out of the door @ModelFeatureFlag(owners = {"vekterli"}, removeAfter="7.last") default boolean useThreePhaseUpdates() { return true; } diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index 9fdca334876..023169a1c4a 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -201,7 +201,6 @@ <i>com.google.inject:guice:jar:no_aop:*:*</i> <i>com.google.j2objc:j2objc-annotations:*:*</i> <i>com.google.protobuf:protobuf-java:*:*</i> - <i>com.microsoft.onnxruntime:onnxruntime:*:*</i> <i>com.sun.activation:javax.activation:*:*</i> <i>com.sun.xml.bind:jaxb-core:*:*</i> <i>com.sun.xml.bind:jaxb-impl:*:*</i> @@ -211,15 +210,10 @@ <i>io.prometheus:simpleclient_common:*:*</i> <i>javax.inject:javax.inject:*:*</i> <i>javax.servlet:javax.servlet-api:*:*</i> - <i>net.java.dev.jna:jna:*:*</i> <i>net.openhft:zero-allocation-hashing:*:*</i> <i>org.antlr:antlr-runtime:*:*</i> <i>org.antlr:antlr4-runtime:*:*</i> - <i>org.apache.commons:commons-compress:*:*</i> - <i>org.apache.commons:commons-exec:*:*</i> - <i>org.apache.commons:commons-math3:*:*</i> <i>org.apache.felix:org.apache.felix.framework:*:*</i> - <i>org.apache.opennlp:opennlp-tools:*:*</i> <i>org.bouncycastle:bcpkix-jdk18on:*:*</i> <i>org.bouncycastle:bcprov-jdk18on:*:*</i> <i>org.bouncycastle:bcutil-jdk18on:*:*</i> diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 6b8428a07ac..328f1b19f10 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -82,6 +82,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private Architecture adminClusterNodeResourcesArchitecture = Architecture.getDefault(); private boolean useRestrictedDataPlaneBindings = false; private Optional<CloudAccount> cloudAccount = Optional.empty(); + private boolean enableDataPlaneFilter = false; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -137,6 +138,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean useTwoPhaseDocumentGc() { return useTwoPhaseDocumentGc; } @Override public boolean useRestrictedDataPlaneBindings() { return useRestrictedDataPlaneBindings; } @Override public Optional<CloudAccount> cloudAccount() { return cloudAccount; } + @Override public boolean enableDataPlaneFilter() { return enableDataPlaneFilter; } public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim; @@ -366,6 +368,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setEnableDataPlaneFilter(boolean enableDataPlaneFilter) { + this.enableDataPlaneFilter = enableDataPlaneFilter; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index 64608e3adea..1971ccc3035 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -24,8 +24,6 @@ public final class ApplicationContainer extends Container implements QrStartConfig.Producer, ZookeeperServerConfig.Producer { - private static final String defaultHostedJVMArgs = "-XX:+SuppressFatalErrorMessage"; - private final boolean isHostedVespa; public ApplicationContainer(AbstractConfigProducer<?> parent, String name, int index, DeployState deployState) { @@ -64,9 +62,6 @@ public final class ApplicationContainer extends Container implements public String getJvmOptions() { StringBuilder b = new StringBuilder(); if (isHostedVespa) { - if (hasDocproc()) { - b.append(ApplicationContainer.defaultHostedJVMArgs).append(' '); - } b.append("-Djdk.tls.server.enableStatusRequestExtension=true ") .append("-Djdk.tls.stapling.responseTimeout=2000 ") .append("-Djdk.tls.stapling.cacheSize=256 ") @@ -79,10 +74,6 @@ public final class ApplicationContainer extends Container implements return b.toString().trim(); } - private boolean hasDocproc() { - return (parent instanceof ContainerCluster) && (((ContainerCluster<?>)parent).getDocproc() != null); - } - @Override public void getConfig(ZookeeperServerConfig.Builder builder) { builder.myid(index()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index f5f25c65cd4..f1b3c74a55d 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -52,6 +52,7 @@ import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; import com.yahoo.vespa.model.container.configserver.ConfigserverCluster; import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import com.yahoo.vespa.model.container.docproc.DocprocChains; +import com.yahoo.vespa.model.container.http.Client; import com.yahoo.vespa.model.container.http.Http; import com.yahoo.vespa.model.container.processing.ProcessingChains; import com.yahoo.vespa.model.container.search.ContainerSearch; @@ -161,6 +162,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> private String jvmGCOptions = null; private boolean deferChangesUntilRestart = false; + private boolean clientsLegacyMode; + private List<Client> clients = List.of(); public ContainerCluster(AbstractConfigProducer<?> parent, String configSubId, String clusterId, DeployState deployState, boolean zooKeeperLocalhostAffinity) { this(parent, configSubId, clusterId, deployState, zooKeeperLocalhostAffinity, 1); @@ -352,6 +355,17 @@ public abstract class ContainerCluster<CONTAINER extends Container> return http; } + public void setClients(boolean legacyMode, List<Client> clients) { + clientsLegacyMode = legacyMode; + this.clients = clients; + } + + public List<Client> getClients() { + return clients; + } + + public boolean clientsLegacyMode() { return clientsLegacyMode; } + public ContainerDocproc getDocproc() { return containerDocproc; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SystemBindingPattern.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SystemBindingPattern.java index 201e26d3575..606557670a5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SystemBindingPattern.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SystemBindingPattern.java @@ -14,6 +14,7 @@ public class SystemBindingPattern extends BindingPattern { public static SystemBindingPattern fromHttpPath(String path) { return new SystemBindingPattern("http", "*", null, path);} public static SystemBindingPattern fromPattern(String binding) { return new SystemBindingPattern(binding);} public static SystemBindingPattern fromHttpPortAndPath(String port, String path) { return new SystemBindingPattern("http", "*", port, path); } + public static SystemBindingPattern fromHttpPortAndPath(int port, String path) { return new SystemBindingPattern("http", "*", Integer.toString(port), path); } @Override public String toString() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Client.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Client.java new file mode 100644 index 00000000000..c851ab2bee6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Client.java @@ -0,0 +1,34 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http; + +import java.security.cert.X509Certificate; +import java.util.List; + +/** + * Represents a client. The client is identified by one of the provided certificates and have a set of permissions. + * + * @author mortent + */ +public class Client { + private String id; + private List<String> permissions; + private List<X509Certificate> certificates; + + public Client(String id, List<String> permissions, List<X509Certificate> certificates) { + this.id = id; + this.permissions = permissions; + this.certificates = certificates; + } + + public String id() { + return id; + } + + public List<String> permissions() { + return permissions; + } + + public List<X509Certificate> certificates() { + return certificates; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudDataPlaneFilter.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudDataPlaneFilter.java new file mode 100644 index 00000000000..6767a61d02b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudDataPlaneFilter.java @@ -0,0 +1,53 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.dependencies.Dependencies; +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.jdisc.http.filter.security.cloud.config.CloudDataPlaneFilterConfig; +import com.yahoo.security.X509CertificateUtils; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; +import com.yahoo.vespa.model.container.http.Client; +import com.yahoo.vespa.model.container.http.Filter; + +import java.util.List; + +class CloudDataPlaneFilter extends Filter implements CloudDataPlaneFilterConfig.Producer { + + private static final String CLASS = "com.yahoo.jdisc.http.filter.security.cloud.CloudDataPlaneFilter"; + private static final String BUNDLE = "jdisc-security-filters"; + + private final ApplicationContainerCluster cluster; + private final boolean legacyMode; + + CloudDataPlaneFilter(ApplicationContainerCluster cluster, boolean legacyMode) { + super(model()); + this.cluster = cluster; + this.legacyMode = legacyMode; + } + + private static ChainedComponentModel model() { + return new ChainedComponentModel( + new BundleInstantiationSpecification( + new ComponentSpecification(CLASS), null, new ComponentSpecification(BUNDLE)), + Dependencies.emptyDependencies()); + } + + @Override + public void getConfig(CloudDataPlaneFilterConfig.Builder builder) { + if (legacyMode) { + builder.legacyMode(true); + } else { + List<Client> clients = cluster.getClients(); + builder.legacyMode(false); + List<CloudDataPlaneFilterConfig.Clients.Builder> clientsList = clients.stream() + .map(x -> new CloudDataPlaneFilterConfig.Clients.Builder() + .id(x.id()) + .certificates(X509CertificateUtils.toPem(x.certificates())) + .permissions(x.permissions())) + .toList(); + builder.clients(clientsList); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 03c9335bbc4..798ca55bdfa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -2,8 +2,13 @@ package com.yahoo.vespa.model.container.xml; import com.google.common.collect.ImmutableList; +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; import com.yahoo.component.Version; +import com.yahoo.component.chain.dependencies.Dependencies; +import com.yahoo.component.chain.model.ChainedComponentModel; import com.yahoo.config.application.Xml; +import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.DeploymentInstanceSpec; @@ -31,8 +36,10 @@ import com.yahoo.config.provision.LoadBalancerSettings; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; +import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.logging.FileConnectionLog; import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.path.Path; import com.yahoo.schema.OnnxModel; import com.yahoo.schema.derived.RankProfileList; import com.yahoo.search.rendering.RendererRegistry; @@ -70,10 +77,14 @@ import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.UserBindingPattern; +import com.yahoo.vespa.model.container.component.chain.Chain; import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import com.yahoo.vespa.model.container.docproc.DocprocChains; import com.yahoo.vespa.model.container.http.AccessControl; +import com.yahoo.vespa.model.container.http.Client; import com.yahoo.vespa.model.container.http.ConnectorFactory; +import com.yahoo.vespa.model.container.http.Filter; +import com.yahoo.vespa.model.container.http.FilterBinding; import com.yahoo.vespa.model.container.http.FilterChains; import com.yahoo.vespa.model.container.http.Http; import com.yahoo.vespa.model.container.http.JettyHttpServer; @@ -88,7 +99,10 @@ import com.yahoo.vespa.model.content.StorageGroup; import org.w3c.dom.Element; import org.w3c.dom.Node; +import java.io.IOException; +import java.io.InputStream; import java.net.URI; +import java.nio.charset.StandardCharsets; import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Arrays; @@ -106,6 +120,7 @@ import java.util.logging.Level; import java.util.regex.Pattern; import java.util.stream.Collectors; +import static com.yahoo.vespa.model.container.ContainerCluster.VIP_HANDLER_BINDING; import static java.util.logging.Level.WARNING; /** @@ -202,6 +217,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addStatusHandlers(cluster, context.getDeployState().isHosted()); addUserHandlers(deployState, cluster, spec, context); + addClients(deployState, spec, cluster, context); addHttp(deployState, spec, cluster, context); addAccessLogs(deployState, cluster, spec); @@ -439,6 +455,86 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addHostedImplicitAccessControlIfNotPresent(deployState, cluster); addDefaultConnectorHostedFilterBinding(cluster); addAdditionalHostedConnector(deployState, cluster, context); + addCloudDataPlaneFilter(deployState, cluster); + } + } + + private static void addCloudDataPlaneFilter(DeployState deployState, ApplicationContainerCluster cluster) { + if (!deployState.isHosted() || !deployState.zone().system().isPublic() || !deployState.featureFlags().enableDataPlaneFilter()) return; + + // Setup secure filter chain + var secureChain = new Chain<Filter>(FilterChains.emptyChainSpec(ComponentId.fromString("cloud-data-plane-secure"))); + secureChain.addInnerComponent(new CloudDataPlaneFilter(cluster, cluster.clientsLegacyMode())); + cluster.getHttp().getFilterChains().add(secureChain); + // Set cloud data plane filter as default request filter chain for data plane connector + cluster.getHttp().getHttpServer().orElseThrow().getConnectorFactories().stream() + .filter(c -> c.getListenPort() == HOSTED_VESPA_DATAPLANE_PORT).findAny().orElseThrow() + .setDefaultRequestFilterChain(secureChain.getComponentId()); + + // Setup insecure filter chain + var insecureChain = new Chain<Filter>(FilterChains.emptyChainSpec(ComponentId.fromString("cloud-data-plane-insecure"))); + insecureChain.addInnerComponent(new Filter( + new ChainedComponentModel( + new BundleInstantiationSpecification( + new ComponentSpecification("com.yahoo.jdisc.http.filter.security.misc.NoopFilter"), + null, new ComponentSpecification("jdisc-security-filters")), + Dependencies.emptyDependencies()))); + cluster.getHttp().getFilterChains().add(insecureChain); + var insecureChainComponentSpec = new ComponentSpecification(insecureChain.getComponentId().toString()); + FilterBinding insecureBinding = + FilterBinding.create(FilterBinding.Type.REQUEST, insecureChainComponentSpec, VIP_HANDLER_BINDING); + cluster.getHttp().getBindings().add(insecureBinding); + // Set insecure filter as default request filter chain for default connector + cluster.getHttp().getHttpServer().orElseThrow().getConnectorFactories().stream() + .filter(c -> c.getListenPort() == Defaults.getDefaults().vespaWebServicePort()).findAny().orElseThrow() + .setDefaultRequestFilterChain(insecureChain.getComponentId()); + + } + + protected void addClients(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) { + if (!deployState.isHosted() || !deployState.zone().system().isPublic() || !deployState.featureFlags().enableDataPlaneFilter()) return; + + List<Client> clients; + Element clientsElement = XML.getChild(spec, "clients"); + boolean legacyMode = false; + if (clientsElement == null) { + Client defaultClient = new Client("default", + List.of(), + getCertificates(app.getFile(Path.fromString("security/clients.pem")))); + clients = List.of(defaultClient); + legacyMode = true; + } else { + clients = XML.getChildren(clientsElement, "client").stream() + .map(this::getCLient) + .toList(); + } + cluster.setClients(legacyMode, clients); + } + + private Client getCLient(Element clientElement) { + String id = XML.attribute("id", clientElement).orElseThrow(); + List<String> permissions = XML.attribute("permissions", clientElement) + .map(p -> p.split(",")).stream() + .flatMap(Arrays::stream) + .toList(); + + List<X509Certificate> x509Certificates = XML.getChildren(clientElement, "certificate").stream() + .map(certElem -> Path.fromString(certElem.getAttribute("file"))) + .map(path -> app.getFile(path)) + .map(this::getCertificates) + .flatMap(Collection::stream) + .toList(); + return new Client(id, permissions, x509Certificates); + } + + private List<X509Certificate> getCertificates(ApplicationFile file) { + try { + InputStream inputStream = file.createInputStream(); + byte[] bytes = inputStream.readAllBytes(); + inputStream.close(); + return X509CertificateUtils.certificateListFromPem(new String(bytes, StandardCharsets.UTF_8)); + } catch (IOException e) { + throw new RuntimeException(e); } } @@ -457,7 +553,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { boolean proxyProtocolMixedMode = deployState.getProperties().featureFlags().enableProxyProtocolMixedMode(); if (deployState.endpointCertificateSecrets().isPresent()) { boolean authorizeClient = deployState.zone().system().isPublic(); - if (authorizeClient && deployState.tlsClientAuthority().isEmpty()) { + List<X509Certificate> clientCertificates = deployState.featureFlags().enableDataPlaneFilter() + ? getClientCertificates(cluster) + : deployState.tlsClientAuthority().map(X509CertificateUtils::certificateListFromPem).orElse(List.of()); + if (authorizeClient && clientCertificates.isEmpty()) { throw new IllegalArgumentException("Client certificate authority security/clients.pem is missing - " + "see: https://cloud.vespa.ai/en/security-model#data-plane"); } @@ -470,7 +569,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { connectorFactory = authorizeClient ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore( - serverName, endpointCertificateSecrets, getTlsClientAuthorities(deployState), tlsCiphersOverride, proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT) + serverName, endpointCertificateSecrets, getTlsClientAuthorities(clientCertificates, deployState), tlsCiphersOverride, proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT) : HostedSslConnectorFactory.withProvidedCertificate( serverName, endpointCertificateSecrets, enforceHandshakeClientAuth, tlsCiphersOverride, proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT); } else { @@ -480,15 +579,21 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { server.addConnector(connectorFactory); } + // Returns the client certificates defined in + private List<X509Certificate> getClientCertificates(ApplicationContainerCluster cluster) { + return cluster.getClients() + .stream() + .map(Client::certificates) + .flatMap(Collection::stream) + .toList(); + } + /* Return trusted certificates as a PEM encoded string containing the concatenation of trusted certs from the application package and all operator certificates. */ - String getTlsClientAuthorities(DeployState deployState) { - List<X509Certificate> trustedCertificates = deployState.tlsClientAuthority() - .map(X509CertificateUtils::certificateListFromPem) - .orElse(Collections.emptyList()); - ArrayList<X509Certificate> x509Certificates = new ArrayList<>(trustedCertificates); + String getTlsClientAuthorities(List<X509Certificate> applicationCertificates, DeployState deployState) { + ArrayList<X509Certificate> x509Certificates = new ArrayList<>(applicationCertificates); x509Certificates.addAll(deployState.getProperties().operatorCertificates()); return X509CertificateUtils.toPem(x509Certificates); } diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 8cd7071462e..938932c3df6 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -24,7 +24,8 @@ ContainerServices = AccessLog* & SecretStore? & ZooKeeper? & - GenericConfig* + GenericConfig* & + Clients? # TODO(ogronnesby): Change this configuration syntax ClientAuthorize = element client-authorize { empty } @@ -128,6 +129,18 @@ Threadpool = element threadpool { element queue-size { xsd:nonNegativeInteger } } +Clients = element clients { + Client* +} + +Client = element client { + ComponentId & + attribute permissions { string } & + element certificate { + attribute file { string } + }+ +} + # SEARCH: SearchInContainer = element search { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 9a2f8a3f0ec..23b576be8b3 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -143,16 +143,8 @@ public class ContainerClusterTest { verifyHeapSizeAsPercentageOfPhysicalMemory(createRoot(!hosted, heapSizeInFlag), !combined, 69, 69); } - private void verifyJvmArgs(boolean isHosted, boolean hasDocproc, String expectedArgs, String jvmArgs) { - if (isHosted && hasDocproc) { - String defaultHostedJVMArgs = "-XX:+SuppressFatalErrorMessage"; - if ( ! "".equals(expectedArgs)) { - defaultHostedJVMArgs = defaultHostedJVMArgs + " "; - } - assertEquals(defaultHostedJVMArgs + expectedArgs, jvmArgs); - } else { - assertEquals(expectedArgs, jvmArgs); - } + private void verifyJvmArgs(boolean isHosted, String expectedArgs, String jvmArgs) { + assertEquals(expectedJvmArgs(isHosted, expectedArgs), jvmArgs); } private void verifyJvmArgs(boolean isHosted, boolean hasDocProc) { @@ -164,15 +156,15 @@ public class ContainerClusterTest { addContainer(root, cluster, "c1", "host-c1"); assertEquals(1, cluster.getContainers().size()); ApplicationContainer container = cluster.getContainers().get(0); - verifyJvmArgs(isHosted, hasDocProc, expectedJvmArgs(isHosted, ""), container.getJvmOptions()); + verifyJvmArgs(isHosted, "", container.getJvmOptions()); container.setJvmOptions("initial"); - verifyJvmArgs(isHosted, hasDocProc, expectedJvmArgs(isHosted, "initial"), container.getJvmOptions()); + verifyJvmArgs(isHosted, "initial", container.getJvmOptions()); container.prependJvmOptions("ignored"); - verifyJvmArgs(isHosted, hasDocProc, expectedJvmArgs(isHosted, "ignored initial"), container.getJvmOptions()); + verifyJvmArgs(isHosted, "ignored initial", container.getJvmOptions()); container.appendJvmOptions("override"); - verifyJvmArgs(isHosted, hasDocProc, expectedJvmArgs(isHosted, "ignored initial override"), container.getJvmOptions()); + verifyJvmArgs(isHosted, "ignored initial override", container.getJvmOptions()); container.setJvmOptions(null); - verifyJvmArgs(isHosted, hasDocProc, expectedJvmArgs(isHosted, ""), container.getJvmOptions()); + verifyJvmArgs(isHosted, "", container.getJvmOptions()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/CloudDataPlaneFilterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/CloudDataPlaneFilterTest.java new file mode 100644 index 00000000000..39d2da11465 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/CloudDataPlaneFilterTest.java @@ -0,0 +1,189 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; +import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.jdisc.http.ServerConfig; +import com.yahoo.jdisc.http.filter.security.cloud.config.CloudDataPlaneFilterConfig; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.X509CertificateUtils; +import com.yahoo.vespa.model.container.ApplicationContainer; +import com.yahoo.vespa.model.container.ContainerModel; +import com.yahoo.vespa.model.container.http.ConnectorFactory; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.w3c.dom.Element; + +import javax.security.auth.x500.X500Principal; +import java.io.File; +import java.io.IOException; +import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class CloudDataPlaneFilterTest extends ContainerModelBuilderTestBase { + + @TempDir + public File applicationFolder; + + Path securityFolder; + private static final String cloudDataPlaneFilterConfigId = "container/filters/chain/cloud-data-plane-secure/component/" + + "com.yahoo.jdisc.http.filter.security.cloud.CloudDataPlaneFilter"; + + @BeforeEach + public void setup() throws IOException { + securityFolder = applicationFolder.toPath().resolve("security"); + Files.createDirectories(securityFolder); + } + + @Test + public void it_generates_correct_config() throws IOException { + Path certFile = securityFolder.resolve("foo.pem"); + Element clusterElem = DomBuilderTest.parse( + """ + <container version='1.0'> + <clients> + <client id="foo" permissions="read,write"> + <certificate file="%s"/> + </client> + </clients> + </container> + """ + .formatted(applicationFolder.toPath().relativize(certFile).toString())); + X509Certificate certificate = createCertificate(certFile); + + buildModel(true, clusterElem); + + CloudDataPlaneFilterConfig config = root.getConfig(CloudDataPlaneFilterConfig.class, cloudDataPlaneFilterConfigId); + assertFalse(config.legacyMode()); + List<CloudDataPlaneFilterConfig.Clients> clients = config.clients(); + assertEquals(1, clients.size()); + CloudDataPlaneFilterConfig.Clients client = clients.get(0); + assertEquals("foo", client.id()); + assertIterableEquals(List.of("read", "write"), client.permissions()); + assertIterableEquals(List.of(X509CertificateUtils.toPem(certificate)), client.certificates()); + + ConnectorConfig connectorConfig = connectorConfig(); + var caCerts = X509CertificateUtils.certificateListFromPem(connectorConfig.ssl().caCertificate()); + assertEquals(1, caCerts.size()); + assertEquals(List.of(certificate), caCerts); + var srvCfg = root.getConfig(ServerConfig.class, "container/http"); + assertEquals("cloud-data-plane-insecure", srvCfg.defaultFilters().get(0).filterId()); + assertEquals(8080, srvCfg.defaultFilters().get(0).localPort()); + assertEquals("cloud-data-plane-secure", srvCfg.defaultFilters().get(1).filterId()); + assertEquals(4443, srvCfg.defaultFilters().get(1).localPort()); + } + + @Test + public void it_generates_correct_legacy_config() throws IOException { + Path certFile = securityFolder.resolve("clients.pem"); + Element clusterElem = DomBuilderTest.parse("<container version='1.0' />"); + X509Certificate certificate = createCertificate(certFile); + + buildModel(true, clusterElem); + + CloudDataPlaneFilterConfig config = root.getConfig(CloudDataPlaneFilterConfig.class, cloudDataPlaneFilterConfigId); + assertTrue(config.legacyMode()); + List<CloudDataPlaneFilterConfig.Clients> clients = config.clients(); + assertEquals(0, clients.size()); + + ConnectorConfig connectorConfig = connectorConfig(); + var caCerts = X509CertificateUtils.certificateListFromPem(connectorConfig.ssl().caCertificate()); + assertEquals(1, caCerts.size()); + assertEquals(List.of(certificate), caCerts); + } + + @Test + public void it_generates_correct_config_when_filter_not_enabled () throws IOException { + Path certFile = securityFolder.resolve("clients.pem"); + Element clusterElem = DomBuilderTest.parse( + """ + <container version='1.0'> + <clients> + <client id="foo" permissions="read,write"> + <certificate file="%s"/> + </client> + </clients> + </container> + """ + .formatted(applicationFolder.toPath().relativize(certFile).toString())); + X509Certificate certificate = createCertificate(certFile); + + buildModel(false, clusterElem); + + // Data plane filter config is not configured + assertFalse(root.getConfigIds().contains("container/component/com.yahoo.jdisc.http.filter.security.cloud.CloudDataPlaneFilter")); + + // Connector config configures ca certs from security/clients.pem + ConnectorConfig connectorConfig = connectorConfig(); + var caCerts = X509CertificateUtils.certificateListFromPem(connectorConfig.ssl().caCertificate()); + assertEquals(1, caCerts.size()); + assertEquals(List.of(certificate), caCerts); + } + + private ConnectorConfig connectorConfig() { + ApplicationContainer container = (ApplicationContainer) root.getProducer("container/container.0"); + List<ConnectorFactory> connectorFactories = container.getHttp().getHttpServer().get().getConnectorFactories(); + ConnectorFactory tlsPort = connectorFactories.stream().filter(connectorFactory -> connectorFactory.getListenPort() == 4443).findFirst().orElseThrow(); + + ConnectorConfig.Builder builder = new ConnectorConfig.Builder(); + tlsPort.getConfig(builder); + + return new ConnectorConfig(builder); + } + + /* + Creates cert, returns + */ + static X509Certificate createCertificate(Path certFile) throws IOException { + KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + X500Principal subject = new X500Principal("CN=mysubject"); + X509Certificate certificate = X509CertificateBuilder + .fromKeypair( + keyPair, subject, Instant.now(), Instant.now().plus(1, ChronoUnit.DAYS), SignatureAlgorithm.SHA512_WITH_ECDSA, BigInteger.valueOf(1)) + .build(); + String certPem = X509CertificateUtils.toPem(certificate); + Files.writeString(certFile, certPem); + return certificate; + } + + public List<ContainerModel> buildModel(boolean enableFilter, Element... clusterElem) { + var applicationPackage = new MockApplicationPackage.Builder() + .withRoot(applicationFolder) + .build(); + + DeployState state = new DeployState.Builder() + .applicationPackage(applicationPackage) + .properties( + new TestProperties() + .setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY"))) + .setHostedVespa(true) + .setEnableDataPlaneFilter(enableFilter)) + .zone(new Zone(SystemName.PublicCd, Environment.dev, RegionName.defaultName())) + .build(); + return createModel(root, state, null, clusterElem); + } +} diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 543f76ca136..7976b1f5524 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -209,6 +209,16 @@ </environment-variables> <node hostalias="host1" /> </nodes> + + <clients> + <client id="client1" permissions="read,write"> + <certificate file="security/file.pem" /> + </client> + <client id="client2" permissions="write"> + <certificate file="security/file1.pem" /> + <certificate file="security/file2.pem" /> + </client> + </clients> </container> <container id='qrsCluster_2' version='1.0'> diff --git a/configdefinitions/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/config/package-info.java b/configdefinitions/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/config/package-info.java new file mode 100644 index 00000000000..2a21b128bb5 --- /dev/null +++ b/configdefinitions/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/config/package-info.java @@ -0,0 +1,6 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +@ExportPackage +package com.yahoo.jdisc.http.filter.security.cloud.config; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/configdefinitions/src/vespa/cloud-data-plane-filter.def b/configdefinitions/src/vespa/cloud-data-plane-filter.def new file mode 100644 index 00000000000..47478a28039 --- /dev/null +++ b/configdefinitions/src/vespa/cloud-data-plane-filter.def @@ -0,0 +1,7 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=jdisc.http.filter.security.cloud.config + +legacyMode bool default=false +clients[].id string +clients[].permissions[] string +clients[].certificates[] string diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java index 01dd47765d2..3bad813ef4b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; -import com.yahoo.compress.ArchiveStreamReader; -import com.yahoo.compress.ArchiveStreamReader.Options; +import com.yahoo.vespa.archive.ArchiveStreamReader; +import com.yahoo.vespa.archive.ArchiveStreamReader.Options; import com.yahoo.vespa.config.server.http.BadRequestException; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.v2.ApplicationApiHandler; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 9750fe7d120..2a19a3b0833 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -210,6 +210,7 @@ public class ModelContextImpl implements ModelContext { private final boolean useRestrictedDataPlaneBindings; private final int heapPercentage; private final boolean useOldJdiscContainerStartup; + private final boolean enableDataPlaneFilter; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -256,6 +257,7 @@ public class ModelContextImpl implements ModelContext { this.useRestrictedDataPlaneBindings = flagValue(source, appId, version, Flags.RESTRICT_DATA_PLANE_BINDINGS); this.heapPercentage = flagValue(source, appId, version, PermanentFlags.HEAP_SIZE_PERCENTAGE); this.useOldJdiscContainerStartup = flagValue(source, appId, version, Flags.USE_OLD_JDISC_CONTAINER_STARTUP); + this.enableDataPlaneFilter = flagValue(source, appId, version, Flags.ENABLE_DATAPLANE_FILTER); } @Override public boolean useOldJdiscContainerStartup() { return useOldJdiscContainerStartup; } @@ -310,6 +312,7 @@ public class ModelContextImpl implements ModelContext { } @Override public boolean useTwoPhaseDocumentGc() { return useTwoPhaseDocumentGc; } @Override public boolean useRestrictedDataPlaneBindings() { return useRestrictedDataPlaneBindings; } + @Override public boolean enableDataPlaneFilter() { return enableDataPlaneFilter; } private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java index 62640668932..903323fcd58 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java @@ -6,6 +6,7 @@ import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.api.Quota; import com.yahoo.config.model.api.TenantSecretStore; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; @@ -19,7 +20,6 @@ import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.NotFoundException; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.tenant.TenantRepository; - import java.security.cert.X509Certificate; import java.time.Instant; import java.util.List; @@ -141,6 +141,10 @@ public abstract class Session implements Comparable<Session> { sessionZooKeeperClient.writeAthenzDomain(athenzDomain); } + public void setQuota(Optional<Quota> quota) { + sessionZooKeeperClient.writeQuota(quota); + } + public void setTenantSecretStores(List<TenantSecretStore> tenantSecretStores) { sessionZooKeeperClient.writeTenantSecretStores(tenantSecretStores); } @@ -180,6 +184,8 @@ public abstract class Session implements Comparable<Session> { public Optional<AthenzDomain> getAthenzDomain() { return sessionZooKeeperClient.readAthenzDomain(); } + public Optional<Quota> getQuota() { return sessionZooKeeperClient.readQuota(); } + public AllocatedHosts getAllocatedHosts() { return sessionZooKeeperClient.getAllocatedHosts(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 6fea1276633..9c50636ecfb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -285,6 +285,7 @@ public class SessionRepository { session.setVespaVersion(existingSession.getVespaVersion()); session.setDockerImageRepository(existingSession.getDockerImageRepository()); session.setAthenzDomain(existingSession.getAthenzDomain()); + session.setQuota(existingSession.getQuota()); session.setTenantSecretStores(existingSession.getTenantSecretStores()); session.setOperatorCertificates(existingSession.getOperatorCertificates()); session.setCloudAccount(existingSession.getCloudAccount()); diff --git a/container-core/pom.xml b/container-core/pom.xml index ed4b05495e3..bdcaab3900b 100644 --- a/container-core/pom.xml +++ b/container-core/pom.xml @@ -84,6 +84,31 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>opennlp-linguistics</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <!-- Pulled in by language-detector in scope compile --> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + <exclusion> + <!-- Pulled in by language-detector in scope compile --> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + <exclusion> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>messagebus</artifactId> <version>${project.version}</version> <exclusions> diff --git a/container-dev/pom.xml b/container-dev/pom.xml index 6847668404b..711afe72c62 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -160,10 +160,6 @@ <artifactId>zero-allocation-hashing</artifactId> </exclusion> <exclusion> - <groupId>org.apache.commons</groupId> - <artifactId>commons-compress</artifactId> - </exclusion> - <exclusion> <groupId>org.lz4</groupId> <artifactId>lz4-java</artifactId> </exclusion> @@ -232,7 +228,7 @@ Excluded artifacts should be added explicitly to the application module to make then visible in users' test classpath. --> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>linguistics</artifactId> + <artifactId>opennlp-linguistics</artifactId> <version>${project.version}</version> <exclusions> <exclusion> diff --git a/container-test/pom.xml b/container-test/pom.xml index 7dcbc794b77..103c118b083 100644 --- a/container-test/pom.xml +++ b/container-test/pom.xml @@ -99,11 +99,6 @@ <artifactId>aircompressor</artifactId> <scope>compile</scope> </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-compress</artifactId> - <scope>compile</scope> - </dependency> <dependency> <!-- TODO: Remove on Vespa 9 --> <!-- not used by Vespa, but was historically on test classpath --> <groupId>org.json</groupId> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java index 53c78d7c8ec..8dbec87828a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java @@ -5,9 +5,9 @@ import com.google.common.hash.Funnel; import com.google.common.hash.Hasher; import com.google.common.hash.Hashing; import com.yahoo.component.Version; -import com.yahoo.compress.ArchiveStreamReader; -import com.yahoo.compress.ArchiveStreamReader.ArchiveFile; -import com.yahoo.compress.ArchiveStreamReader.Options; +import com.yahoo.vespa.archive.ArchiveStreamReader; +import com.yahoo.vespa.archive.ArchiveStreamReader.ArchiveFile; +import com.yahoo.vespa.archive.ArchiveStreamReader.Options; import com.yahoo.config.application.FileSystemWrapper; import com.yahoo.config.application.FileSystemWrapper.FileWrapper; import com.yahoo.config.application.XmlPreProcessor; @@ -27,7 +27,6 @@ import com.yahoo.vespa.hosted.controller.deployment.ZipBuilder; import com.yahoo.yolean.Exceptions; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.nio.file.NoSuchFileException; @@ -38,7 +37,6 @@ import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java index 185c97f866e..d22a41f74a4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java @@ -1,27 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application.pkg; -import com.yahoo.compress.ArchiveStreamReader; -import com.yahoo.compress.ArchiveStreamReader.ArchiveFile; -import com.yahoo.compress.ArchiveStreamReader.Options; +import com.yahoo.vespa.archive.ArchiveStreamReader; +import com.yahoo.vespa.archive.ArchiveStreamReader.ArchiveFile; +import com.yahoo.vespa.archive.ArchiveStreamReader.Options; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.List; -import java.util.Map; -import java.util.Map.Entry; import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; import java.util.function.Predicate; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; -import java.util.zip.ZipOutputStream; /** * A list of entries read from a ZIP archive, and their contents. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java index 526bf0c53b3..64dd51ce304 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java @@ -138,6 +138,7 @@ public class Notifier { .map(Notifier::linkify) .map(m -> "<li>" + m + "</li>") .collect(Collectors.joining())) + .replace("[[LINK_TO_NOTIFICATION]]", notificationLink(notification.source())) .replace("[[LINK_TO_ACCOUNT_NOTIFICATIONS]]", accountNotificationsUri(content.notification().source().tenant())) .replace("[[LINK_TO_PRIVACY_POLICY]]", "https://legal.yahoo.com/xw/en/yahoo/privacy/topic/b2bprivacypolicy/index.html") .replace("[[LINK_TO_TERMS_OF_SERVICE]]", consoleUri("terms-of-service-trial.html")) @@ -170,4 +171,23 @@ public class Notifier { return new UriBuilder(dashboardUri).append(path).toString(); } + private String notificationLink(NotificationSource source) { + var uri = new UriBuilder(dashboardUri); + uri = uri.append("tenant").append(source.tenant().value()); + if (source.application().isPresent()) + uri = uri.append("application").append(source.application().get().value()); + if (source.isProduction()) { + uri = uri.append("prod/instance"); + if (source.jobType().isPresent()) { + uri = uri.append(source.instance().get().value()); + } + } + else { + uri = uri.append("dev/instance/").append(source.instance().get().value()); + } + if (source.jobType().isPresent()) { + uri = uri.append("job").append(source.jobType().get().jobName()).append("run").append(String.valueOf(source.runNumber().getAsLong())); + } + return uri.toString(); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java index 2b4c6411386..c1e3ec851f6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java @@ -36,7 +36,6 @@ import java.security.Principal; import java.security.cert.X509Certificate; import java.time.Instant; import java.util.Optional; -import java.util.OptionalInt; import java.util.Scanner; import java.util.function.Function; @@ -176,7 +175,7 @@ public class ControllerApiHandler extends AuditLoggingRequestHandler { } private HttpResponse overrideConfidence(HttpRequest request, String version) { - Confidence confidence = Confidence.valueOf(asString(request.getData())); + Confidence confidence = Confidence.valueOf(asString(request.getData()).trim()); maintenance.upgrader().overrideConfidence(Version.fromString(version), confidence); return new UpgraderResponse(maintenance.upgrader()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java index 28fa5183e5c..f9add356f19 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java @@ -23,7 +23,7 @@ public class UpgraderResponse extends SlimeJsonResponse { Cursor array = root.setArray("confidenceOverrides"); upgrader.confidenceOverrides().forEach((version, confidence) -> { Cursor object = array.addObject(); - object.setString(version.toString(), confidence.name()); + object.setString(version.toFullString(), confidence.name()); }); return slime; diff --git a/controller-server/src/main/resources/mail/mail-notification.tmpl b/controller-server/src/main/resources/mail/mail-notification.tmpl index 868a3a76bab..8c02c05723f 100644 --- a/controller-server/src/main/resources/mail/mail-notification.tmpl +++ b/controller-server/src/main/resources/mail/mail-notification.tmpl @@ -486,7 +486,7 @@ valign="middle" > <a - href="#" + href="[[LINK_TO_NOTIFICATION]]" style=" display: inline-block; background: #005a8e; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java index 80bcbc7ee7e..799189410ea 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java @@ -96,19 +96,19 @@ public class ControllerApiTest extends ControllerContainerTest { // Override confidence tester.assertResponse( operatorRequest("http://localhost:8080/controller/v1/jobs/upgrader/confidence/6.42", "broken", Request.Method.POST), - "{\"upgradesPerMinute\":42.0,\"confidenceOverrides\":[{\"6.42\":\"broken\"}]}", + "{\"upgradesPerMinute\":42.0,\"confidenceOverrides\":[{\"6.42.0\":\"broken\"}]}", 200); // Override confidence for another version tester.assertResponse( - operatorRequest("http://localhost:8080/controller/v1/jobs/upgrader/confidence/6.43", "broken", Request.Method.POST), - "{\"upgradesPerMinute\":42.0,\"confidenceOverrides\":[{\"6.42\":\"broken\"},{\"6.43\":\"broken\"}]}", + operatorRequest("http://localhost:8080/controller/v1/jobs/upgrader/confidence/6.43", " broken ", Request.Method.POST), + "{\"upgradesPerMinute\":42.0,\"confidenceOverrides\":[{\"6.42.0\":\"broken\"},{\"6.43.0\":\"broken\"}]}", 200); // Remove first override tester.assertResponse( operatorRequest("http://localhost:8080/controller/v1/jobs/upgrader/confidence/6.42", "", Request.Method.DELETE), - "{\"upgradesPerMinute\":42.0,\"confidenceOverrides\":[{\"6.43\":\"broken\"}]}", + "{\"upgradesPerMinute\":42.0,\"confidenceOverrides\":[{\"6.43.0\":\"broken\"}]}", 200); assertFalse(tester.controller().auditLogger().readLog().entries().isEmpty(), "Actions are logged to audit log"); diff --git a/controller-server/src/test/resources/mail/notification.txt b/controller-server/src/test/resources/mail/notification.txt index 5ca26cb4753..9f1583d7674 100644 --- a/controller-server/src/test/resources/mail/notification.txt +++ b/controller-server/src/test/resources/mail/notification.txt @@ -486,7 +486,7 @@ valign="middle" > <a - href="#" + href="https://dashboard.tld/tenant/tenant1/application/default/prod/instance" style=" display: inline-block; background: #005a8e; diff --git a/fat-model-dependencies/pom.xml b/fat-model-dependencies/pom.xml index 9801098d5b6..ff77b5e9601 100644 --- a/fat-model-dependencies/pom.xml +++ b/fat-model-dependencies/pom.xml @@ -73,6 +73,13 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>model-integration</artifactId> <version>${project.version}</version> + <exclusions> + <exclusion> + <!-- OPTIMIZATION: very large (60 MB) and not needed for config generation --> + <groupId>com.microsoft.onnxruntime</groupId> + <artifactId>onnxruntime</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -93,6 +100,16 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>org.apache.commons</groupId> + <artifactId>commons-exec</artifactId> + </exclusion> + <exclusion> + <groupId>net.java.dev.jna</groupId> + <artifactId>jna</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -118,6 +135,13 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>container-core</artifactId> <version>${project.version}</version> + <exclusions> + <exclusion> + <!-- OPTIMIZATION: large (10 MB) and not needed for config generation --> + <groupId>com.yahoo.vespa</groupId> + <artifactId>opennlp-linguistics</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -154,6 +178,11 @@ <groupId>com.ibm.icu</groupId> <artifactId>icu4j</artifactId> </exclusion> + <exclusion> + <!-- OPTIMIZATION: large (4 MB) and only used for query dispatch TDistribution--> + <groupId>org.apache.commons</groupId> + <artifactId>commons-math3</artifactId> + </exclusion> </exclusions> </dependency> <dependency> diff --git a/filedistribution/pom.xml b/filedistribution/pom.xml index 7916698b62c..97b17cc00d8 100644 --- a/filedistribution/pom.xml +++ b/filedistribution/pom.xml @@ -22,7 +22,7 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>container-apache-http-client-bundle</artifactId> <version>${project.version}</version> - <scope>compile</scope> + <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -34,11 +34,13 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>jrt</artifactId> <version>${project.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -55,6 +57,7 @@ <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>junit</groupId> @@ -64,6 +67,7 @@ <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-compress</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>io.airlift</groupId> @@ -72,6 +76,7 @@ <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.slf4j</groupId> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 2f8efd5a717..a53f13e1303 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -424,6 +424,13 @@ public class Flags { "If true, use the old vespa-start-container-daemon script.", "Takes effect immediately?", ZONE_ID, APPLICATION_ID); + + public static final UnboundBooleanFlag ENABLE_DATAPLANE_FILTER = defineFeatureFlag( + "enable-dataplane-filter", false, + List.of("tokle", "bjorncs"), "2022-11-15", "2023-01-31", + "Setup data plane filter from clients configuration", + "Takes effect on redeployment", + APPLICATION_ID); public static final UnboundBooleanFlag USE_LOCKS_IN_FILEDISTRIBUTION = defineFeatureFlag( "use-locks-in-filedistribution", false, diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudDataPlaneFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudDataPlaneFilter.java new file mode 100644 index 00000000000..04446ddd4de --- /dev/null +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudDataPlaneFilter.java @@ -0,0 +1,146 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.filter.security.cloud; + +import com.yahoo.component.annotation.Inject; +import com.yahoo.container.jdisc.AclMapping; +import com.yahoo.container.jdisc.RequestHandlerSpec; +import com.yahoo.container.jdisc.RequestView; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; +import com.yahoo.jdisc.http.filter.security.cloud.config.CloudDataPlaneFilterConfig; +import com.yahoo.security.X509CertificateUtils; + +import java.security.Principal; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import static com.yahoo.jdisc.http.filter.security.cloud.CloudDataPlaneFilter.Permission.READ; +import static com.yahoo.jdisc.http.filter.security.cloud.CloudDataPlaneFilter.Permission.WRITE; + +/** + * @author bjorncs + */ +public class CloudDataPlaneFilter extends JsonSecurityRequestFilterBase { + + private static final Logger log = Logger.getLogger(CloudDataPlaneFilter.class.getName()); + + private final boolean legacyMode; + private final List<Client> allowedClients; + + @Inject + public CloudDataPlaneFilter(CloudDataPlaneFilterConfig cfg) { + this.legacyMode = cfg.legacyMode(); + if (legacyMode) { + allowedClients = List.of(); + log.fine(() -> "Legacy mode enabled"); + } else { + allowedClients = parseClients(cfg); + } + } + + private static List<Client> parseClients(CloudDataPlaneFilterConfig cfg) { + Set<String> ids = new HashSet<>(); + List<Client> clients = new ArrayList<>(cfg.clients().size()); + for (var c : cfg.clients()) { + if (ids.contains(c.id())) + throw new IllegalArgumentException("Clients definition has duplicate id '%s'".formatted(c.id())); + ids.add(c.id()); + List<X509Certificate> certs; + try { + certs = c.certificates().stream().map(X509CertificateUtils::fromPem).toList(); + } catch (Exception e) { + throw new IllegalArgumentException( + "Client '%s' contains invalid X.509 certificate PEM: %s".formatted(c.id(), e.toString()), e); + } + EnumSet<Permission> permissions = c.permissions().stream().map(Permission::of) + .collect(Collectors.toCollection(() -> EnumSet.noneOf(Permission.class))); + clients.add(new Client(c.id(), permissions, certs)); + } + if (clients.isEmpty()) throw new IllegalArgumentException("Empty clients configuration"); + log.fine(() -> "Configured clients with ids %s".formatted(ids)); + return clients; + } + + @Override + protected Optional<ErrorResponse> filter(DiscFilterRequest req) { + var certs = req.getClientCertificateChain(); + log.fine(() -> "Certificate chain contains %d elements".formatted(certs.size())); + if (certs.isEmpty()) { + log.fine("Missing client certificate"); + return Optional.of(new ErrorResponse(Response.Status.UNAUTHORIZED, "Unauthorized")); + } + if (legacyMode) { + log.fine("Legacy mode validation complete"); + req.setUserPrincipal(new ClientPrincipal(Set.of(), Set.of(READ, WRITE))); + return Optional.empty(); + } + RequestView view = req.asRequestView(); + var permission = Optional.ofNullable((RequestHandlerSpec) req.getAttribute(RequestHandlerSpec.ATTRIBUTE_NAME)) + .or(() -> Optional.of(RequestHandlerSpec.DEFAULT_INSTANCE)) + .flatMap(spec -> { + var action = spec.aclMapping().get(view); + var maybePermission = Permission.of(action); + if (maybePermission.isEmpty()) log.fine(() -> "Unknown action '%s'".formatted(action)); + return maybePermission; + }).orElse(null); + if (permission == null) { + log.fine(() -> "No valid permission mapping defined for %s @ '%s'".formatted(view.method(), view.uri())); + return Optional.of(new ErrorResponse(Response.Status.FORBIDDEN, "Forbidden")); + } + var clientCert = certs.get(0); + var clientIds = new TreeSet<String>(); + var permissions = new TreeSet<Permission>(); + for (Client c : allowedClients) { + if (c.permissions().contains(permission) && c.certificates().contains(clientCert)) { + clientIds.add(c.id()); + permissions.addAll(c.permissions()); + } + } + log.fine(() -> "Client with ids=%s, permissions=%s" + .formatted(clientIds, permissions.stream().map(Permission::asString).toList())); + if (clientIds.isEmpty()) return Optional.of(new ErrorResponse(Response.Status.FORBIDDEN, "Forbidden")); + req.setUserPrincipal(new ClientPrincipal(clientIds, permissions)); + return Optional.empty(); + } + + public record ClientPrincipal(Set<String> ids, Set<Permission> permissions) implements Principal { + public ClientPrincipal { ids = Set.copyOf(ids); permissions = Set.copyOf(permissions); } + @Override public String getName() { + return "ids=%s,permissions=%s".formatted(ids, permissions.stream().map(Permission::asString).toList()); + } + } + + enum Permission { READ, WRITE; + String asString() { + return switch (this) { + case READ -> "read"; + case WRITE -> "write"; + }; + } + static Permission of(String v) { + return switch (v) { + case "read" -> READ; + case "write" -> WRITE; + default -> throw new IllegalArgumentException("Invalid permission '%s'".formatted(v)); + }; + } + static Optional<Permission> of(AclMapping.Action a) { + if (a.equals(AclMapping.Action.READ)) return Optional.of(READ); + if (a.equals(AclMapping.Action.WRITE)) return Optional.of(WRITE); + return Optional.empty(); + } + } + + private record Client(String id, EnumSet<Permission> permissions, List<X509Certificate> certificates) { + Client { permissions = EnumSet.copyOf(permissions); certificates = List.copyOf(certificates); } + } +} diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/package-info.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/package-info.java new file mode 100644 index 00000000000..a4b2a23ea95 --- /dev/null +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/package-info.java @@ -0,0 +1,8 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package com.yahoo.jdisc.http.filter.security.cloud; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cloud/CloudDataPlaneFilterTest.java b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cloud/CloudDataPlaneFilterTest.java new file mode 100644 index 00000000000..d8b6312e90e --- /dev/null +++ b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cloud/CloudDataPlaneFilterTest.java @@ -0,0 +1,151 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.filter.security.cloud; + +import com.yahoo.container.jdisc.AclMapping.Action; +import com.yahoo.container.jdisc.HttpMethodAclMapping; +import com.yahoo.container.jdisc.RequestHandlerSpec; +import com.yahoo.container.jdisc.RequestHandlerTestDriver.MockResponseHandler; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.jdisc.http.filter.security.cloud.CloudDataPlaneFilter.ClientPrincipal; +import com.yahoo.jdisc.http.filter.security.cloud.config.CloudDataPlaneFilterConfig; +import com.yahoo.jdisc.http.filter.util.FilterTestUtils; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.X509CertificateUtils; +import org.junit.jupiter.api.Test; + +import javax.security.auth.x500.X500Principal; +import java.math.BigInteger; +import java.security.cert.X509Certificate; +import java.util.List; +import java.util.Set; + +import static com.yahoo.jdisc.Response.Status.FORBIDDEN; +import static com.yahoo.jdisc.Response.Status.UNAUTHORIZED; +import static com.yahoo.jdisc.http.filter.security.cloud.CloudDataPlaneFilter.Permission.READ; +import static com.yahoo.jdisc.http.filter.security.cloud.CloudDataPlaneFilter.Permission.WRITE; +import static com.yahoo.security.KeyAlgorithm.EC; +import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; +import static java.time.Instant.EPOCH; +import static java.time.temporal.ChronoUnit.DAYS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +/** + * @author bjorncs + */ +class CloudDataPlaneFilterTest { + + private static final X509Certificate FEED_CERT = certificate("my-feed-client"); + private static final X509Certificate SEARCH_CERT = certificate("my-search-client"); + private static final X509Certificate LEGACY_CLIENT = certificate("my-legacy-client"); + private static final String FEED_CLIENT_ID = "feed-client"; + private static final String SEARCH_CLIENT_ID = "search-client"; + + @Test + void accepts_any_trusted_client_certificate_in_legacy_mode() { + var req = FilterTestUtils.newRequestBuilder().withClientCertificate(LEGACY_CLIENT).build(); + var responseHandler = new MockResponseHandler(); + newFilterWithLegacyMode().filter(req, responseHandler); + assertNull(responseHandler.getResponse()); + assertEquals(new ClientPrincipal(Set.of(), Set.of(READ, WRITE)), req.getUserPrincipal()); + } + + @Test + void fails_on_missing_certificate_in_legacy_mode() { + var req = FilterTestUtils.newRequestBuilder().build(); + var responseHandler = new MockResponseHandler(); + newFilterWithLegacyMode().filter(req, responseHandler); + assertNotNull(responseHandler.getResponse()); + assertEquals(UNAUTHORIZED, responseHandler.getResponse().getStatus()); + } + + @Test + void accepts_client_with_valid_certificate() { + var req = FilterTestUtils.newRequestBuilder() + .withMethod(Method.POST) + .withClientCertificate(FEED_CERT) + .build(); + var responseHandler = new MockResponseHandler(); + newFilterWithClientsConfig().filter(req, responseHandler); + assertNull(responseHandler.getResponse()); + assertEquals(new ClientPrincipal(Set.of(FEED_CLIENT_ID), Set.of(WRITE)), req.getUserPrincipal()); + } + + @Test + void fails_on_client_with_invalid_permissions() { + var req = FilterTestUtils.newRequestBuilder() + .withMethod(Method.POST) + .withClientCertificate(SEARCH_CERT) + .build(); + var responseHandler = new MockResponseHandler(); + newFilterWithClientsConfig().filter(req, responseHandler); + assertNotNull(responseHandler.getResponse()); + assertEquals(FORBIDDEN, responseHandler.getResponse().getStatus()); + } + + @Test + void supports_handler_with_custom_request_spec() { + // Spec that maps POST as action 'read' + var spec = RequestHandlerSpec.builder() + .withAclMapping(HttpMethodAclMapping.standard() + .override(Method.POST, Action.READ).build()) + .build(); + var req = FilterTestUtils.newRequestBuilder() + .withMethod(Method.POST) + .withClientCertificate(SEARCH_CERT) + .withAttribute(RequestHandlerSpec.ATTRIBUTE_NAME, spec) + .build(); + var responseHandler = new MockResponseHandler(); + newFilterWithClientsConfig().filter(req, responseHandler); + assertNull(responseHandler.getResponse()); + assertEquals(new ClientPrincipal(Set.of(SEARCH_CLIENT_ID), Set.of(READ)), req.getUserPrincipal()); + } + + @Test + void fails_on_handler_with_custom_request_spec_with_invalid_action() { + // Spec that maps POST as action 'read' + var spec = RequestHandlerSpec.builder() + .withAclMapping(HttpMethodAclMapping.standard() + .override(Method.GET, Action.custom("custom")).build()) + .build(); + var req = FilterTestUtils.newRequestBuilder() + .withMethod(Method.GET) + .withClientCertificate(SEARCH_CERT) + .withAttribute(RequestHandlerSpec.ATTRIBUTE_NAME, spec) + .build(); + var responseHandler = new MockResponseHandler(); + newFilterWithClientsConfig().filter(req, responseHandler); + assertNotNull(responseHandler.getResponse()); + assertEquals(FORBIDDEN, responseHandler.getResponse().getStatus()); + } + + private static CloudDataPlaneFilter newFilterWithLegacyMode() { + return new CloudDataPlaneFilter(new CloudDataPlaneFilterConfig.Builder().legacyMode(true).build()); + } + + private static CloudDataPlaneFilter newFilterWithClientsConfig() { + return new CloudDataPlaneFilter( + new CloudDataPlaneFilterConfig.Builder() + .clients(List.of( + new CloudDataPlaneFilterConfig.Clients.Builder() + .certificates(X509CertificateUtils.toPem(FEED_CERT)) + .permissions(WRITE.asString()) + .id(FEED_CLIENT_ID), + new CloudDataPlaneFilterConfig.Clients.Builder() + .certificates(X509CertificateUtils.toPem(SEARCH_CERT)) + .permissions(READ.asString()) + .id(SEARCH_CLIENT_ID))) + .build()); + } + + private static X509Certificate certificate(String name) { + var key = KeyUtils.generateKeypair(EC); + var subject = new X500Principal("CN=%s".formatted(name)); + return X509CertificateBuilder + .fromKeypair(key, subject, EPOCH, EPOCH.plus(1, DAYS), SHA256_WITH_ECDSA, BigInteger.ONE).build(); + } + + +}
\ No newline at end of file diff --git a/linguistics-components/pom.xml b/linguistics-components/pom.xml index eae50c778b5..ad4cbd6ce22 100644 --- a/linguistics-components/pom.xml +++ b/linguistics-components/pom.xml @@ -43,6 +43,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>opennlp-linguistics</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>config-bundle</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/linguistics/pom.xml b/linguistics/pom.xml index 87ecc35ba2c..bfbf1beeaea 100644 --- a/linguistics/pom.xml +++ b/linguistics/pom.xml @@ -58,10 +58,6 @@ <scope>provided</scope> <classifier>no_aop</classifier> </dependency> - <dependency> - <groupId>org.apache.opennlp</groupId> - <artifactId>opennlp-tools</artifactId> - </dependency> </dependencies> <build> <plugins> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java index fb789874acf..e0e11ad0a3a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java @@ -19,15 +19,17 @@ public class Container extends PartialContainer { private final ContainerResources resources; private final int conmonPid; private final List<Network> networks; + private final List<String> createCommand; public Container(ContainerId id, ContainerName name, Instant createdAt, State state, String imageId, DockerImage image, Map<String, String> labels, int pid, int conmonPid, String hostname, - ContainerResources resources, List<Network> networks, boolean managed) { + ContainerResources resources, List<Network> networks, boolean managed, List<String> createCommand) { super(id, name, createdAt, state, imageId, image, labels, pid, managed); this.hostname = Objects.requireNonNull(hostname); this.resources = Objects.requireNonNull(resources); this.conmonPid = conmonPid; this.networks = List.copyOf(Objects.requireNonNull(networks)); + this.createCommand = List.copyOf(Objects.requireNonNull(createCommand)); } /** The hostname of this, if any */ @@ -50,18 +52,23 @@ public class Container extends PartialContainer { return networks; } + /** The command used to create this */ + public List<String> createCommand() { + return createCommand; + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; if (!super.equals(o)) return false; - Container that = (Container) o; - return conmonPid == that.conmonPid && hostname.equals(that.hostname) && resources.equals(that.resources) && networks.equals(that.networks); + Container container = (Container) o; + return conmonPid == container.conmonPid && hostname.equals(container.hostname) && resources.equals(container.resources) && networks.equals(container.networks) && createCommand.equals(container.createCommand); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), hostname, resources, conmonPid, networks); + return Objects.hash(super.hashCode(), hostname, resources, conmonPid, networks, createCommand); } /** The network of a container */ diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java index 2aa1d12c491..630509cf482 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java @@ -35,6 +35,9 @@ public interface ContainerEngine { /** Remove given container. The container will be stopped if necessary */ void removeContainer(TaskContext context, PartialContainer container); + /** Returns whether the given container should be re-created to apply new configuration */ + boolean shouldRecreate(NodeAgentContext context, Container container, ContainerResources wantedResources); + /** Get container for given context */ Optional<Container> getContainer(NodeAgentContext context); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java index 9060261b806..e632acd5223 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java @@ -57,6 +57,10 @@ public class ContainerOperations { containerEngine.updateContainer(context, containerId, containerResources); } + public boolean shouldRecreate(NodeAgentContext context, Container container, ContainerResources wantedResources) { + return containerEngine.shouldRecreate(context, container, wantedResources); + } + public Optional<Container> getContainer(NodeAgentContext context) { return containerEngine.getContainer(context); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 20ea29381f3..d46325e4bca 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -332,34 +332,34 @@ public class NodeAgentImpl implements NodeAgent { } private List<String> shouldRemoveContainer(NodeAgentContext context, Container existingContainer) { - final NodeState nodeState = context.node().state(); + NodeState nodeState = context.node().state(); List<String> reasons = new ArrayList<>(); - if (nodeState == NodeState.dirty || nodeState == NodeState.provisioned) + if (nodeState == NodeState.dirty || nodeState == NodeState.provisioned) { reasons.add("Node in state " + nodeState + ", container should no longer be running"); - + } if (context.node().wantedDockerImage().isPresent() && !context.node().wantedDockerImage().get().equals(existingContainer.image())) { - reasons.add("The node is supposed to run a new Docker image: " + reasons.add("The node is supposed to run a new image: " + existingContainer.image().asString() + " -> " + context.node().wantedDockerImage().get().asString()); } - - if (!existingContainer.state().isRunning()) + if (!existingContainer.state().isRunning()) { reasons.add("Container no longer running"); - + } if (currentRebootGeneration < context.node().wantedRebootGeneration()) { reasons.add(String.format("Container reboot wanted. Current: %d, Wanted: %d", currentRebootGeneration, context.node().wantedRebootGeneration())); } - ContainerResources wantedContainerResources = getContainerResources(context); if (!wantedContainerResources.equalsMemory(existingContainer.resources())) { reasons.add("Container should be running with different memory allocation, wanted: " + wantedContainerResources.toStringMemory() + ", actual: " + existingContainer.resources().toStringMemory()); } - - if (containerState == STARTING) + if (containerOperations.shouldRecreate(context, existingContainer, wantedContainerResources)) { + reasons.add("Container should be re-created to apply new configuration"); + } + if (containerState == STARTING) { reasons.add("Container failed to start"); - + } return reasons; } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java index 2d3a4976fe5..744d78d83da 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java @@ -28,6 +28,7 @@ public class ContainerEngineMock implements ContainerEngine { private final Map<ContainerName, Container> containers = new ConcurrentHashMap<>(); private final Map<String, ImageDownload> images = new ConcurrentHashMap<>(); + private boolean asyncImageDownload = false; public ContainerEngineMock asyncImageDownload(boolean enabled) { @@ -112,6 +113,11 @@ public class ContainerEngineMock implements ContainerEngine { } @Override + public boolean shouldRecreate(NodeAgentContext context, Container container, ContainerResources wantedResources) { + return false; + } + + @Override public void updateContainer(NodeAgentContext context, ContainerId containerId, ContainerResources containerResources) { Container container = requireContainer(context.containerName()); containers.put(container.name(), new Container(containerId, container.name(), container.createdAt(), container.state(), @@ -119,7 +125,8 @@ public class ContainerEngineMock implements ContainerEngine { container.labels(), container.pid(), container.conmonPid(), container.hostname(), containerResources, container.networks(), - container.managed())); + container.managed(), + container.createCommand())); } @Override @@ -200,7 +207,8 @@ public class ContainerEngineMock implements ContainerEngine { context.hostname().value(), containerResources, List.of(), - true); + true, + List.of()); } private static class ImageDownload { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java index 9a5ca8c805e..8fb65e4bd47 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java @@ -59,7 +59,7 @@ public class ContainerOperationsTest { private Container createContainer(String name, boolean managed) { return new Container(new ContainerId("id-of-" + name), new ContainerName(name), Instant.EPOCH, PartialContainer.State.running, "image-id", DockerImage.EMPTY, Map.of(), 42, 43, name, - ContainerResources.UNLIMITED, List.of(), managed); + ContainerResources.UNLIMITED, List.of(), managed, List.of()); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java index 2ef6780dff6..0d83a397d33 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java @@ -123,7 +123,7 @@ public class ContainerImagePrunerTest { return new Container(new ContainerId("id-of-" + name), new ContainerName(name), Instant.EPOCH, Container.State.running, imageId, DockerImage.EMPTY, Map.of(), 42, 43, name + ".example.com", ContainerResources.UNLIMITED, - List.of(), true); + List.of(), true, List.of()); } private static class Tester { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index fb132c9b717..6e980b26cdf 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -358,6 +358,15 @@ public class NodeAgentImplTest { verify(orchestrator, times(1)).resume(eq(hostName)); verify(nodeRepository, times(1)).updateNodeAttributes(eq(hostName), eq(new NodeAttributes() .withRebootGeneration(wantedRebootGeneration))); + + // Re-create if new container config needs to be applied + when(containerOperations.shouldRecreate(eq(context), any(), any())).thenReturn(true); + nodeAgent.doConverge(context); + verify(containerOperations, times(2)).removeContainer(eq(context), any()); + verify(containerOperations, times(2)).createContainer(eq(context), any()); + verify(orchestrator, times(2)).resume(eq(hostName)); + verify(nodeRepository, times(2)).updateNodeAttributes(eq(hostName), eq(new NodeAttributes() + .withRebootGeneration(wantedRebootGeneration))); } @Test @@ -815,7 +824,8 @@ public class NodeAgentImplTest { hostName, containerResources, List.of(), - true)) : + true, + List.of())) : Optional.empty(); }).when(containerOperations).getContainer(any()); } diff --git a/opennlp-linguistics/abi-spec.json b/opennlp-linguistics/abi-spec.json new file mode 100644 index 00000000000..6f31cf5a2e6 --- /dev/null +++ b/opennlp-linguistics/abi-spec.json @@ -0,0 +1 @@ +{ }
\ No newline at end of file diff --git a/opennlp-linguistics/pom.xml b/opennlp-linguistics/pom.xml new file mode 100644 index 00000000000..40f1e95f4f4 --- /dev/null +++ b/opennlp-linguistics/pom.xml @@ -0,0 +1,83 @@ +<?xml version="1.0"?> +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>8-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>opennlp-linguistics</artifactId> + <packaging>container-plugin</packaging> + <version>8-SNAPSHOT</version> + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>component</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>annotations</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>configdefinitions</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>linguistics</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.google.inject</groupId> + <artifactId>guice</artifactId> + <scope>provided</scope> + <classifier>no_aop</classifier> + </dependency> + <dependency> + <groupId>org.apache.opennlp</groupId> + <artifactId>opennlp-tools</artifactId> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>abi-check-plugin</artifactId> + </plugin> + </plugins> + </build> +</project> diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/DefaultLanguageDetectorContextGenerator.java b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/DefaultLanguageDetectorContextGenerator.java index 27c23d8d3e6..27c23d8d3e6 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/DefaultLanguageDetectorContextGenerator.java +++ b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/DefaultLanguageDetectorContextGenerator.java diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java index 0cf4634c6c3..0cf4634c6c3 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java +++ b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java index d7a7d3a4744..d7a7d3a4744 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java +++ b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java index 1d96d8a0cdf..1d96d8a0cdf 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java +++ b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java index 8080dc92729..8080dc92729 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java +++ b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java index 883319e2f8b..883319e2f8b 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java +++ b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/VespaCharSequenceNormalizer.java b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/VespaCharSequenceNormalizer.java index df8f3fad520..df8f3fad520 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/VespaCharSequenceNormalizer.java +++ b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/VespaCharSequenceNormalizer.java diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/WordCharDetector.java b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/WordCharDetector.java index d7e3f88ae8d..d7e3f88ae8d 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/WordCharDetector.java +++ b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/WordCharDetector.java diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/package-info.java b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/package-info.java index 9606578b3ac..9606578b3ac 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/package-info.java +++ b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/package-info.java diff --git a/linguistics/src/main/resources/models/langdetect-183.bin b/opennlp-linguistics/src/main/resources/models/langdetect-183.bin Binary files differindex c3cde217050..c3cde217050 100644 --- a/linguistics/src/main/resources/models/langdetect-183.bin +++ b/opennlp-linguistics/src/main/resources/models/langdetect-183.bin diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java b/opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java index 746ed10da1c..746ed10da1c 100644 --- a/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java +++ b/opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java b/opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java index cd2a0f73895..cd2a0f73895 100644 --- a/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java +++ b/opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java b/opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java index a8c637bc6ec..a8c637bc6ec 100644 --- a/linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java +++ b/opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java @@ -91,6 +91,7 @@ <module>jdisc_core_test</module> <module>jrt</module> <module>linguistics</module> + <module>opennlp-linguistics</module> <module>linguistics-components</module> <module>logd</module> <module>logserver</module> diff --git a/vespajlib/pom.xml b/vespajlib/pom.xml index 4c57b615c16..d903fb5ec0d 100644 --- a/vespajlib/pom.xml +++ b/vespajlib/pom.xml @@ -36,10 +36,6 @@ <artifactId>aircompressor</artifactId> </dependency> <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-compress</artifactId> - </dependency> - <dependency> <groupId>net.openhft</groupId> <artifactId>zero-allocation-hashing</artifactId> <exclusions> |