From b3d356dcfa20bee655e4e95531f82268c68c32d1 Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Mon, 12 Aug 2019 16:36:14 +0200 Subject: Add the CA certificates to configuration and read it from application package --- config-model-api/abi-spec.json | 2 ++ .../config/application/api/ApplicationPackage.java | 6 +++++- .../java/com/yahoo/config/model/api/TlsSecrets.java | 16 ++++++++-------- .../com/yahoo/config/model/deploy/DeployState.java | 19 +++++++++++++++++++ .../vespa/model/container/ApplicationContainer.java | 19 ++++++++++++++----- .../model/container/ApplicationContainerCluster.java | 6 ++++++ .../http/ssl/ConfiguredDirectSslProvider.java | 5 ++++- .../model/container/xml/ContainerModelBuilder.java | 6 +++--- .../model/container/xml/ContainerServiceBuilder.java | 9 ++++++++- .../java/com/yahoo/vespa/model/content/Content.java | 2 +- .../vespa/model/container/ContainerClusterTest.java | 2 +- .../vespa/config/server/deploy/ZooKeeperClient.java | 3 +++ jdisc_http_service/abi-spec.json | 2 ++ .../configdefinitions/jdisc.http.connector.def | 7 +++++++ 14 files changed, 83 insertions(+), 21 deletions(-) diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 9f86fe4dea2..8d83d16a272 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -111,6 +111,7 @@ "public java.util.List getQueryProfileFiles()", "public java.util.List getQueryProfileTypeFiles()", "public java.util.List getPageTemplateFiles()", + "public com.yahoo.config.application.api.ApplicationFile getClientSecurityFile()", "public abstract java.lang.String getHostSource()", "public abstract java.lang.String getServicesSource()", "public abstract java.util.Optional getDeployment()", @@ -148,6 +149,7 @@ "public static final com.yahoo.path.Path RULES_DIR", "public static final com.yahoo.path.Path DEPLOYMENT_FILE", "public static final com.yahoo.path.Path VALIDATION_OVERRIDES", + "public static final com.yahoo.path.Path SECURITY_DIR", "public static final java.lang.String SD_NAME_SUFFIX", "public static final java.lang.String RANKEXPRESSION_NAME_SUFFIX", "public static final java.lang.String RULES_NAME_SUFFIX", diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index 06f8034453d..5cd119dcf65 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -27,7 +27,6 @@ import java.util.Map; import java.util.Optional; import java.util.jar.JarEntry; import java.util.jar.JarFile; -import java.util.stream.Collectors; /** * Represents an application package, that is, used as input when creating a VespaModel and as @@ -73,6 +72,8 @@ public interface ApplicationPackage { Path DEPLOYMENT_FILE = Path.fromString("deployment.xml"); Path VALIDATION_OVERRIDES = Path.fromString("validation-overrides.xml"); + Path SECURITY_DIR = Path.fromString("security"); + String SD_NAME_SUFFIX = ".sd"; String RANKEXPRESSION_NAME_SUFFIX = ".expression"; String RULES_NAME_SUFFIX = ".sr"; @@ -178,6 +179,9 @@ public interface ApplicationPackage { /** Does {@link #getFiles} on the page template directory and gets all xml files */ default List getPageTemplateFiles() { return getFiles(PAGE_TEMPLATES_DIR,".xml"); } + /** Returns handle for the file containing client certificate authorities */ + default ApplicationFile getClientSecurityFile() { return getFile(SECURITY_DIR.append("clients.pem")); } + //For generating error messages String getHostSource(); String getServicesSource(); diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java b/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java index 3cb4cedcbac..6a8b5a237ab 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java @@ -1,30 +1,30 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.api; - public class TlsSecrets { +public class TlsSecrets { public static final TlsSecrets MISSING = new TlsSecrets(); - private final String certificate; + private final String certificate; private final String key; - private TlsSecrets() { - this(null,null); + private TlsSecrets() { + this(null, null); } - public TlsSecrets(String certificate, String key) { + public TlsSecrets(String certificate, String key) { this.certificate = certificate; this.key = key; } - public String certificate() { + public String certificate() { return certificate; } - public String key() { + public String key() { return key; } - public boolean isMissing() { + public boolean isMissing() { return this == MISSING; } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 977946cbf71..46e380e1ebd 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -24,6 +24,7 @@ import com.yahoo.config.model.provision.SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.Zone; +import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.SearchBuilder; @@ -39,8 +40,10 @@ import com.yahoo.vespa.model.container.search.SemanticRules; import com.yahoo.vespa.model.search.SearchDefinition; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.Reader; +import java.io.UncheckedIOException; import java.time.Instant; import java.util.Collection; import java.util.Collections; @@ -263,6 +266,22 @@ public class DeployState implements ConfigDefinitionStore { public Optional tlsSecrets() { return properties.tlsSecrets(); } + public Optional tlsClientAuthority() { + var caFile = applicationPackage.getClientSecurityFile(); + if (caFile.exists()) { + try { + var caPem = IOUtils.readAll(caFile.createReader()); + return Optional.of(caPem); + } catch (FileNotFoundException e) { + return Optional.empty(); + } catch (IOException e) { + throw new UncheckedIOException("Failed reading certificate from application: " + caFile.getPath(), e); + } + } else { + return Optional.empty(); + } + } + public static class Builder { private ApplicationPackage applicationPackage = MockApplicationPackage.createEmpty(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index 48f7fa3c1a2..f00b617de46 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -22,11 +22,11 @@ public final class ApplicationContainer extends Container { private final boolean isHostedVespa; - public ApplicationContainer(AbstractConfigProducer parent, String name, int index, boolean isHostedVespa, Optional tlsSecrets) { - this(parent, name, false, index, isHostedVespa, tlsSecrets); + public ApplicationContainer(AbstractConfigProducer parent, String name, int index, boolean isHostedVespa, Optional tlsSecrets, Optional tlsCa) { + this(parent, name, false, index, isHostedVespa, tlsSecrets, tlsCa); } - public ApplicationContainer(AbstractConfigProducer parent, String name, boolean retired, int index, boolean isHostedVespa, Optional tlsSecrets) { + public ApplicationContainer(AbstractConfigProducer parent, String name, boolean retired, int index, boolean isHostedVespa, Optional tlsSecrets, Optional tlsCa) { super(parent, name, retired, index); this.isHostedVespa = isHostedVespa; @@ -36,8 +36,17 @@ public final class ApplicationContainer extends Container { JettyHttpServer server = Optional.ofNullable(getHttp()) .map(Http::getHttpServer) .orElse(getDefaultHttpServer()); - server.addConnector(new ConnectorFactory(connectorName, 4443, - new ConfiguredDirectSslProvider(server.getComponentId().getName(), tlsSecrets.get().key(), tlsSecrets.get().certificate(), null, null))); + + var sslProvider = new ConfiguredDirectSslProvider( + server.getComponentId().getName(), + tlsSecrets.get().key(), + tlsSecrets.get().certificate(), + null, + tlsCa.orElse(null), + null + ); + + server.addConnector(new ConnectorFactory(connectorName, 4443, sslProvider)); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 473971c5e7a..5656299b302 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -56,6 +56,7 @@ public final class ApplicationContainerCluster extends ContainerCluster tlsSecrets; + private Optional tlsClientAuthority; private final boolean enableGroupingSessionCache; private MbusParams mbusParams; @@ -65,6 +66,7 @@ public final class ApplicationContainerCluster extends ContainerCluster(this, "rest-api"); servletGroup = new ConfigProducerGroup<>(this, "servlet"); @@ -183,6 +185,10 @@ public final class ApplicationContainerCluster extends ContainerCluster getTlsClientAuthority() { + return tlsClientAuthority; + } + public boolean enableGroupingSessionCache() { return enableGroupingSessionCache; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredDirectSslProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredDirectSslProvider.java index 28dba3331d3..b47aa501ece 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredDirectSslProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredDirectSslProvider.java @@ -26,9 +26,10 @@ public class ConfiguredDirectSslProvider extends SimpleComponent implements Conn private final String privateKey; private final String certificate; private final String caCertificatePath; + private final String caCertificate; private final ConnectorConfig.Ssl.ClientAuth.Enum clientAuthentication; - public ConfiguredDirectSslProvider(String servername, String privateKey, String certificate, String caCertificatePath, String clientAuthentication) { + public ConfiguredDirectSslProvider(String servername, String privateKey, String certificate, String caCertificatePath, String caCertificate, String clientAuthentication) { super(new ComponentModel( new BundleInstantiationSpecification(new ComponentId(COMPONENT_ID_PREFIX+servername), fromString(COMPONENT_CLASS), @@ -36,6 +37,7 @@ public class ConfiguredDirectSslProvider extends SimpleComponent implements Conn this.privateKey = privateKey; this.certificate = certificate; this.caCertificatePath = caCertificatePath; + this.caCertificate = caCertificate; this.clientAuthentication = mapToConfigEnum(clientAuthentication); } @@ -45,6 +47,7 @@ public class ConfiguredDirectSslProvider extends SimpleComponent implements Conn builder.ssl.privateKey(privateKey); builder.ssl.certificate(certificate); builder.ssl.caCertificateFile(Optional.ofNullable(caCertificatePath).orElse("")); + builder.ssl.caCertificate(Optional.ofNullable(caCertificate).orElse("")); builder.ssl.clientAuth(clientAuthentication); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 2bfb1da9dcb..a7a4e0fb540 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -440,7 +440,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder { } private void addStandaloneNode(ApplicationContainerCluster cluster) { - ApplicationContainer container = new ApplicationContainer(cluster, "standalone", cluster.getContainers().size(), cluster.isHostedVespa(), cluster.getTlsSecrets()); + ApplicationContainer container = new ApplicationContainer(cluster, "standalone", cluster.getContainers().size(), cluster.isHostedVespa(), cluster.getTlsSecrets(), cluster.getTlsClientAuthority()); cluster.addContainers(Collections.singleton(container)); } @@ -506,7 +506,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder { Element nodesElement = XML.getChild(containerElement, "nodes"); Element rotationsElement = XML.getChild(containerElement, "rotations"); if (nodesElement == null) { // default single node on localhost - ApplicationContainer node = new ApplicationContainer(cluster, "container.0", 0, cluster.isHostedVespa(), cluster.getTlsSecrets()); + ApplicationContainer node = new ApplicationContainer(cluster, "container.0", 0, cluster.isHostedVespa(), cluster.getTlsSecrets(), cluster.getTlsClientAuthority()); HostResource host = allocateSingleNodeHost(cluster, log, containerElement, context); node.setHostResource(host); node.initService(context.getDeployLogger()); @@ -695,7 +695,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder { List nodes = new ArrayList<>(); for (Map.Entry entry : hosts.entrySet()) { String id = "container." + entry.getValue().index(); - ApplicationContainer container = new ApplicationContainer(cluster, id, entry.getValue().retired(), entry.getValue().index(), cluster.isHostedVespa(), cluster.getTlsSecrets()); + ApplicationContainer container = new ApplicationContainer(cluster, id, entry.getValue().retired(), entry.getValue().index(), cluster.isHostedVespa(), cluster.getTlsSecrets(), cluster.getTlsClientAuthority()); container.setHostResource(entry.getKey()); container.initService(deployLogger); nodes.add(container); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java index 46271d3c0a2..c976a7fb153 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java @@ -22,7 +22,14 @@ public class ContainerServiceBuilder extends VespaDomBuilder.DomConfigProducerBu @Override protected ApplicationContainer doBuild(DeployState deployState, AbstractConfigProducer parent, Element nodeElem) { - return new ApplicationContainer(parent, id, index, deployState.isHosted(), deployState.tlsSecrets()); + return new ApplicationContainer( + parent, + id, + index, + deployState.isHosted(), + deployState.tlsSecrets(), + deployState.tlsClientAuthority() + ); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index 8eda707be99..f028f0ac0cc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -324,7 +324,7 @@ public class Content extends ConfigModel { if (!processedHosts.contains(host)) { String containerName = String.valueOf(searchNode.getDistributionKey()); ApplicationContainer docprocService = new ApplicationContainer(indexingCluster, containerName, index, - modelContext.getDeployState().isHosted(), modelContext.getDeployState().tlsSecrets()); + modelContext.getDeployState().isHosted(), modelContext.getDeployState().tlsSecrets(), Optional.empty()); index++; docprocService.useDynamicPorts(); docprocService.setHostResource(host); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index ac85a958ed5..89169c44079 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -252,7 +252,7 @@ public class ContainerClusterTest { } private static void addContainer(DeployLogger deployLogger, ApplicationContainerCluster cluster, String name, String hostName) { - ApplicationContainer container = new ApplicationContainer(cluster, name, 0, cluster.isHostedVespa(), cluster.getTlsSecrets()); + ApplicationContainer container = new ApplicationContainer(cluster, name, 0, cluster.isHostedVespa(), cluster.getTlsSecrets(), cluster.getTlsClientAuthority()); container.setHostResource(new HostResource(new Host(null, hostName))); container.initService(deployLogger); cluster.addContainer(container); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index 484124991d9..a99a2a2917d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -202,6 +202,9 @@ public class ZooKeeperClient { writeDir(app.getFile(ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR), getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR), true); + writeDir(app.getFile(ApplicationPackage.SECURITY_DIR), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.SECURITY_DIR), + true); } private void writeDir(ApplicationFile file, Path zooKeeperAppPath, boolean recurse) throws IOException { diff --git a/jdisc_http_service/abi-spec.json b/jdisc_http_service/abi-spec.json index f915dc1e8c1..b06250c4593 100644 --- a/jdisc_http_service/abi-spec.json +++ b/jdisc_http_service/abi-spec.json @@ -82,6 +82,7 @@ "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder certificateFile(java.lang.String)", "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder certificate(java.lang.String)", "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder caCertificateFile(java.lang.String)", + "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder caCertificate(java.lang.String)", "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder clientAuth(com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum)", "public com.yahoo.jdisc.http.ConnectorConfig$Ssl build()" ], @@ -137,6 +138,7 @@ "public java.lang.String certificateFile()", "public java.lang.String certificate()", "public java.lang.String caCertificateFile()", + "public java.lang.String caCertificate()", "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum clientAuth()" ], "fields": [] diff --git a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def index c6c6fad345b..9b6fb5401e2 100644 --- a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def +++ b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def @@ -69,7 +69,14 @@ ssl.certificateFile string default="" ssl.certificate string default="" # with trusted CA certificates in PEM format. Used to verify clients +# - this is the name of a file on the local container file system +# - only one of caCertificateFile and caCertificate ssl.caCertificateFile string default="" +# with trusted CA certificates in PEM format. Used to verify clients +# - this is the actual certificates instead of a pointer to the file +# - only one of caCertificateFile and caCertificate +ssl.caCertificate string default="" + # Client authentication mode. See SSLEngine.getNeedClientAuth()/getWantClientAuth() for details. ssl.clientAuth enum { DISABLED, WANT_AUTH, NEED_AUTH } default=DISABLED -- cgit v1.2.3 From f68fae7329b21c87793909c1638b210a6784ca3e Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Tue, 13 Aug 2019 14:04:23 +0200 Subject: Create ability to use JimFS in MockApplicationPackage --- config-model/pom.xml | 6 ++ .../config/model/test/MockApplicationPackage.java | 94 +++++++++++----------- .../RankingExpressionWithTensorFlowTestCase.java | 4 +- 3 files changed, 57 insertions(+), 47 deletions(-) diff --git a/config-model/pom.xml b/config-model/pom.xml index 180a2e40921..0684f857581 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -45,6 +45,12 @@ guava-testlib test + + com.google.jimfs + jimfs + 1.1 + compile + commons-io commons-io diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 60435decb04..f1120ce517f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -17,24 +17,26 @@ import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.config.application.api.ApplicationPackage; -import java.io.BufferedInputStream; import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.Reader; import java.io.StringReader; import java.io.UncheckedIOException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +import com.google.common.jimfs.Jimfs; + /** * For testing purposes only * @@ -48,7 +50,7 @@ public class MockApplicationPackage implements ApplicationPackage { public static final String MUSIC_SEARCHDEFINITION = createSearchDefinition("music", "foo"); public static final String BOOK_SEARCHDEFINITION = createSearchDefinition("book", "bar"); - private final File root; + private final java.nio.file.Path root; private final String hostsS; private final String servicesS; private final List searchDefinitions; @@ -59,7 +61,7 @@ public class MockApplicationPackage implements ApplicationPackage { private final QueryProfileRegistry queryProfileRegistry; private final ApplicationMetaData applicationMetaData; - protected MockApplicationPackage(File root, String hosts, String services, List searchDefinitions, + protected MockApplicationPackage(java.nio.file.Path root, String hosts, String services, List searchDefinitions, String searchDefinitionDir, String deploymentSpec, String validationOverrides, boolean failOnValidateXml, String queryProfile, String queryProfileType) { @@ -77,7 +79,7 @@ public class MockApplicationPackage implements ApplicationPackage { } /** Returns the root of this application package relative to the current dir */ - protected File root() { return root; } + protected java.nio.file.Path root() { return root; } @Override public String getApplicationName() { @@ -129,7 +131,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public ApplicationFile getFile(Path file) { - return new MockApplicationFile(file, Path.fromString(root.toString())); + return new MockApplicationFile(file, root); } @Override @@ -190,7 +192,8 @@ public class MockApplicationPackage implements ApplicationPackage { public static class Builder { - private File root = new File("nonexisting"); + private FileSystem fileSystem = FileSystems.getDefault(); + private java.nio.file.Path root = fileSystem.getPath("nonexisting"); private String hosts = null; private String services = null; private List searchDefinitions = Collections.emptyList(); @@ -205,7 +208,13 @@ public class MockApplicationPackage implements ApplicationPackage { } public Builder withRoot(File root) { - this.root = root; + this.root = this.fileSystem.getPath(root.getName()); + return this; + } + + public Builder withInMemoryFileSystem() { + this.fileSystem = Jimfs.newFileSystem(); + this.root = this.fileSystem.getPath("/"); return this; } @@ -319,32 +328,32 @@ public class MockApplicationPackage implements ApplicationPackage { public static class MockApplicationFile extends ApplicationFile { /** The path to the application package root */ - private final Path root; + private final java.nio.file.Path root; /** The File pointing to the actual file represented by this */ - private final File file; + private final java.nio.file.Path file; - public MockApplicationFile(Path filePath, Path applicationPackagePath) { + public MockApplicationFile(Path filePath, java.nio.file.Path applicationPackagePath) { super(filePath); this.root = applicationPackagePath; - file = applicationPackagePath.append(filePath).toFile(); + file = applicationPackagePath.resolve(path.toString()); } @Override public boolean isDirectory() { - return file.isDirectory(); + return Files.isDirectory(file); } @Override public boolean exists() { - return file.exists(); + return Files.exists(file); } @Override public Reader createReader() { try { if ( ! exists()) throw new FileNotFoundException("File '" + file + "' does not exist"); - return IOUtils.createReader(file, "UTF-8"); + return Files.newBufferedReader(file); } catch (IOException e) { throw new UncheckedIOException(e); @@ -355,7 +364,7 @@ public class MockApplicationPackage implements ApplicationPackage { public InputStream createInputStream() { try { if ( ! exists()) throw new FileNotFoundException("File '" + file + "' does not exist"); - return new BufferedInputStream(new FileInputStream(file)); + return Files.newInputStream(file); } catch (IOException e) { throw new UncheckedIOException(e); @@ -364,14 +373,18 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public ApplicationFile createDirectory() { - file.mkdirs(); + try { + Files.createDirectories(file); + } catch (IOException e) { + throw new UncheckedIOException(e); + } return this; } @Override public ApplicationFile writeFile(Reader input) { try { - IOUtils.writeFile(file, IOUtils.readAll(input), false); + Files.writeString(file, IOUtils.readAll(input), StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE); return this; } catch (IOException e) { @@ -382,7 +395,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public ApplicationFile appendFile(String value) { try { - IOUtils.writeFile(file, value, true); + Files.writeString(file, value, StandardOpenOption.APPEND); return this; } catch (IOException e) { @@ -393,15 +406,25 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public List listFiles(PathFilter filter) { if ( ! isDirectory()) return Collections.emptyList(); - return Arrays.stream(file.listFiles()).filter(f -> filter.accept(Path.fromString(f.toString()))) - .map(f -> new MockApplicationFile(asApplicationRelativePath(f), root)) - .collect(Collectors.toList()); + try { + return Files.list(file) + .map(f -> Path.fromString(f.toString())) + .filter(filter::accept) + .map(f -> new MockApplicationFile(f, root)) + .collect(Collectors.toList()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } @Override public ApplicationFile delete() { - file.delete(); - return this; + try { + Files.delete(file); + return this; + } catch (IOException e) { + throw new UncheckedIOException(e); + } } @Override @@ -413,25 +436,6 @@ public class MockApplicationPackage implements ApplicationPackage { public int compareTo(ApplicationFile other) { return this.getPath().getName().compareTo((other).getPath().getName()); } - - /** Strips the application package root path prefix from the path of the given file */ - private Path asApplicationRelativePath(File file) { - Path path = Path.fromString(file.toString()); - - Iterator pathIterator = path.iterator(); - // Skip the path elements this shares with the root - for (Iterator rootIterator = root.iterator(); rootIterator.hasNext(); ) { - String rootElement = rootIterator.next(); - String pathElement = pathIterator.next(); - if ( ! rootElement.equals(pathElement)) throw new RuntimeException("Assumption broken"); - } - // Build a path from the remaining - Path relative = Path.fromString(""); - while (pathIterator.hasNext()) - relative = relative.append(pathIterator.next()); - return relative; - } - } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java index 38515c36690..3e415ea67e4 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java @@ -474,14 +474,14 @@ public class RankingExpressionWithTensorFlowTestCase { } StoringApplicationPackage(Path applicationPackageWritableRoot, String queryProfile, String queryProfileType) { - super(new File(applicationPackageWritableRoot.toString()), + super(java.nio.file.Path.of(applicationPackageWritableRoot.toString()), null, null, Collections.emptyList(), null, null, null, false, queryProfile, queryProfileType); } @Override public ApplicationFile getFile(Path file) { - return new MockApplicationFile(file, Path.fromString(root().toString())); + return new MockApplicationFile(file, root()); } @Override -- cgit v1.2.3 From 8e9e48809ded9fe9178e620b7985f5e63ccf570d Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Tue, 13 Aug 2019 14:05:40 +0200 Subject: Check that we are configured to use certificates to validate --- .../vespa/model/container/ApplicationContainerCluster.java | 5 +++++ .../vespa/model/container/xml/ContainerModelBuilder.java | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 5656299b302..2b9cd18d64b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -57,6 +57,7 @@ public final class ApplicationContainerCluster extends ContainerCluster tlsSecrets; private Optional tlsClientAuthority; + private boolean useTlsClientAuthority = false; private final boolean enableGroupingSessionCache; private MbusParams mbusParams; @@ -211,6 +212,10 @@ public final class ApplicationContainerCluster extends ContainerCluster { addClientProviders(deployState, spec, cluster); addServerProviders(deployState, spec, cluster); + addTlsClientAuthority(deployState, spec, cluster); + addAthensCopperArgos(cluster, context); // Must be added after nodes. } + private void addTlsClientAuthority(DeployState deployState, Element spec, ApplicationContainerCluster cluster) { + var clientAuthorized = XML.getChild(spec, "client-authorize"); + if (clientAuthorized != null && deployState.tlsClientAuthority().isEmpty()) { + if (deployState.tlsClientAuthority().isEmpty()) { + throw new RuntimeException("client-authorize set, but security/clients.pem is missing"); + } + cluster.useTlsClientAuthority(true); + } + } + private void addSecretStore(ApplicationContainerCluster cluster, Element spec) { Element secretStoreElement = XML.getChild(spec, "secret-store"); if (secretStoreElement != null) { -- cgit v1.2.3 From ef0fea3965f1c58d27e9ca767a8fc47137aec91b Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Tue, 13 Aug 2019 14:06:09 +0200 Subject: Test that we a) get certificate in config, and b) respect the services.xml config --- .../container/xml/ContainerModelBuilderTest.java | 39 ++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 8b92e1091ca..29fe6c3e9a8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -25,6 +25,7 @@ import com.yahoo.container.servlet.ServletConfigConfig; import com.yahoo.container.usability.BindingsOverviewHandler; import com.yahoo.jdisc.http.ServletPathsConfig; import com.yahoo.net.HostName; +import com.yahoo.path.Path; import com.yahoo.prelude.cluster.QrMonitorConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.AbstractService; @@ -39,9 +40,12 @@ import org.junit.Test; import org.w3c.dom.Element; import org.xml.sax.SAXException; +import java.io.File; import java.io.IOException; +import java.io.StringReader; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.logging.Level; import java.util.stream.Collectors; @@ -644,6 +648,41 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { assertEquals("env1", secretStore.getGroups().get(0).environment); } + @Test + public void client_ca_carts_fail_with_missing_clients_pem() { + Element clusterElem = DomBuilderTest.parse( + "", + " ", + ""); + try { + createModel(root, clusterElem); + } catch (RuntimeException e) { + assertEquals(e.getMessage(), "client-authorize set, but security/clients.pem is missing"); + return; + } + fail(); + } + + @Test + public void client_ca_carts_succeeds_with_client_authorize_and_clients_pem() { + var applicationPackage = new MockApplicationPackage.Builder() + .withInMemoryFileSystem() + .build(); + + applicationPackage.getFile(Path.fromString("security")).createDirectory(); + applicationPackage.getFile(Path.fromString("security/clients.pem")).writeFile(new StringReader("I am a very nice certificate")); + + var deployState = DeployState.createTestState(applicationPackage); + + Element clusterElem = DomBuilderTest.parse( + "", + " ", + ""); + + createModel(root, deployState, null, clusterElem); + assertEquals(Optional.of("I am a very nice certificate"), getContainerCluster("container").getTlsClientAuthority()); + } + @Test public void environment_vars_are_honoured() { Element clusterElem = DomBuilderTest.parse( -- cgit v1.2.3 From 89e823a26a879194fd8a1cff09745baf63428cc3 Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Wed, 14 Aug 2019 11:24:03 +0200 Subject: Revert "Create ability to use JimFS in MockApplicationPackage" This reverts commit f68fae7329b21c87793909c1638b210a6784ca3e. --- config-model/pom.xml | 6 -- .../config/model/test/MockApplicationPackage.java | 94 +++++++++++----------- .../RankingExpressionWithTensorFlowTestCase.java | 4 +- 3 files changed, 47 insertions(+), 57 deletions(-) diff --git a/config-model/pom.xml b/config-model/pom.xml index 0684f857581..180a2e40921 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -45,12 +45,6 @@ guava-testlib test - - com.google.jimfs - jimfs - 1.1 - compile - commons-io commons-io diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index f1120ce517f..60435decb04 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -17,26 +17,24 @@ import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.config.application.api.ApplicationPackage; +import java.io.BufferedInputStream; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.Reader; import java.io.StringReader; import java.io.UncheckedIOException; -import java.nio.file.FileSystem; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.StandardOpenOption; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; -import com.google.common.jimfs.Jimfs; - /** * For testing purposes only * @@ -50,7 +48,7 @@ public class MockApplicationPackage implements ApplicationPackage { public static final String MUSIC_SEARCHDEFINITION = createSearchDefinition("music", "foo"); public static final String BOOK_SEARCHDEFINITION = createSearchDefinition("book", "bar"); - private final java.nio.file.Path root; + private final File root; private final String hostsS; private final String servicesS; private final List searchDefinitions; @@ -61,7 +59,7 @@ public class MockApplicationPackage implements ApplicationPackage { private final QueryProfileRegistry queryProfileRegistry; private final ApplicationMetaData applicationMetaData; - protected MockApplicationPackage(java.nio.file.Path root, String hosts, String services, List searchDefinitions, + protected MockApplicationPackage(File root, String hosts, String services, List searchDefinitions, String searchDefinitionDir, String deploymentSpec, String validationOverrides, boolean failOnValidateXml, String queryProfile, String queryProfileType) { @@ -79,7 +77,7 @@ public class MockApplicationPackage implements ApplicationPackage { } /** Returns the root of this application package relative to the current dir */ - protected java.nio.file.Path root() { return root; } + protected File root() { return root; } @Override public String getApplicationName() { @@ -131,7 +129,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public ApplicationFile getFile(Path file) { - return new MockApplicationFile(file, root); + return new MockApplicationFile(file, Path.fromString(root.toString())); } @Override @@ -192,8 +190,7 @@ public class MockApplicationPackage implements ApplicationPackage { public static class Builder { - private FileSystem fileSystem = FileSystems.getDefault(); - private java.nio.file.Path root = fileSystem.getPath("nonexisting"); + private File root = new File("nonexisting"); private String hosts = null; private String services = null; private List searchDefinitions = Collections.emptyList(); @@ -208,13 +205,7 @@ public class MockApplicationPackage implements ApplicationPackage { } public Builder withRoot(File root) { - this.root = this.fileSystem.getPath(root.getName()); - return this; - } - - public Builder withInMemoryFileSystem() { - this.fileSystem = Jimfs.newFileSystem(); - this.root = this.fileSystem.getPath("/"); + this.root = root; return this; } @@ -328,32 +319,32 @@ public class MockApplicationPackage implements ApplicationPackage { public static class MockApplicationFile extends ApplicationFile { /** The path to the application package root */ - private final java.nio.file.Path root; + private final Path root; /** The File pointing to the actual file represented by this */ - private final java.nio.file.Path file; + private final File file; - public MockApplicationFile(Path filePath, java.nio.file.Path applicationPackagePath) { + public MockApplicationFile(Path filePath, Path applicationPackagePath) { super(filePath); this.root = applicationPackagePath; - file = applicationPackagePath.resolve(path.toString()); + file = applicationPackagePath.append(filePath).toFile(); } @Override public boolean isDirectory() { - return Files.isDirectory(file); + return file.isDirectory(); } @Override public boolean exists() { - return Files.exists(file); + return file.exists(); } @Override public Reader createReader() { try { if ( ! exists()) throw new FileNotFoundException("File '" + file + "' does not exist"); - return Files.newBufferedReader(file); + return IOUtils.createReader(file, "UTF-8"); } catch (IOException e) { throw new UncheckedIOException(e); @@ -364,7 +355,7 @@ public class MockApplicationPackage implements ApplicationPackage { public InputStream createInputStream() { try { if ( ! exists()) throw new FileNotFoundException("File '" + file + "' does not exist"); - return Files.newInputStream(file); + return new BufferedInputStream(new FileInputStream(file)); } catch (IOException e) { throw new UncheckedIOException(e); @@ -373,18 +364,14 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public ApplicationFile createDirectory() { - try { - Files.createDirectories(file); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + file.mkdirs(); return this; } @Override public ApplicationFile writeFile(Reader input) { try { - Files.writeString(file, IOUtils.readAll(input), StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE); + IOUtils.writeFile(file, IOUtils.readAll(input), false); return this; } catch (IOException e) { @@ -395,7 +382,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public ApplicationFile appendFile(String value) { try { - Files.writeString(file, value, StandardOpenOption.APPEND); + IOUtils.writeFile(file, value, true); return this; } catch (IOException e) { @@ -406,25 +393,15 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public List listFiles(PathFilter filter) { if ( ! isDirectory()) return Collections.emptyList(); - try { - return Files.list(file) - .map(f -> Path.fromString(f.toString())) - .filter(filter::accept) - .map(f -> new MockApplicationFile(f, root)) - .collect(Collectors.toList()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + return Arrays.stream(file.listFiles()).filter(f -> filter.accept(Path.fromString(f.toString()))) + .map(f -> new MockApplicationFile(asApplicationRelativePath(f), root)) + .collect(Collectors.toList()); } @Override public ApplicationFile delete() { - try { - Files.delete(file); - return this; - } catch (IOException e) { - throw new UncheckedIOException(e); - } + file.delete(); + return this; } @Override @@ -436,6 +413,25 @@ public class MockApplicationPackage implements ApplicationPackage { public int compareTo(ApplicationFile other) { return this.getPath().getName().compareTo((other).getPath().getName()); } + + /** Strips the application package root path prefix from the path of the given file */ + private Path asApplicationRelativePath(File file) { + Path path = Path.fromString(file.toString()); + + Iterator pathIterator = path.iterator(); + // Skip the path elements this shares with the root + for (Iterator rootIterator = root.iterator(); rootIterator.hasNext(); ) { + String rootElement = rootIterator.next(); + String pathElement = pathIterator.next(); + if ( ! rootElement.equals(pathElement)) throw new RuntimeException("Assumption broken"); + } + // Build a path from the remaining + Path relative = Path.fromString(""); + while (pathIterator.hasNext()) + relative = relative.append(pathIterator.next()); + return relative; + } + } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java index 3e415ea67e4..38515c36690 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java @@ -474,14 +474,14 @@ public class RankingExpressionWithTensorFlowTestCase { } StoringApplicationPackage(Path applicationPackageWritableRoot, String queryProfile, String queryProfileType) { - super(java.nio.file.Path.of(applicationPackageWritableRoot.toString()), + super(new File(applicationPackageWritableRoot.toString()), null, null, Collections.emptyList(), null, null, null, false, queryProfile, queryProfileType); } @Override public ApplicationFile getFile(Path file) { - return new MockApplicationFile(file, root()); + return new MockApplicationFile(file, Path.fromString(root().toString())); } @Override -- cgit v1.2.3 From 0e76a74167984c7f3b13b61e75f7e0a98a4fbf95 Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Wed, 14 Aug 2019 11:30:36 +0200 Subject: Use temporary folder rule for application directory --- .../yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 29fe6c3e9a8..e4fcf6305ed 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -36,7 +36,9 @@ import com.yahoo.vespa.model.container.SecretStore; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.w3c.dom.Element; import org.xml.sax.SAXException; @@ -76,6 +78,8 @@ import static org.junit.Assert.fail; * @author gjoranv */ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { + @Rule + public TemporaryFolder applicationFolder = new TemporaryFolder(); @Test public void deprecated_jdisc_tag_is_allowed() { @@ -666,7 +670,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test public void client_ca_carts_succeeds_with_client_authorize_and_clients_pem() { var applicationPackage = new MockApplicationPackage.Builder() - .withInMemoryFileSystem() + .withRoot(applicationFolder.getRoot()) .build(); applicationPackage.getFile(Path.fromString("security")).createDirectory(); -- cgit v1.2.3 From 6db214234310b8e2841d964c610cf5dcf01b747b Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Wed, 14 Aug 2019 14:13:46 +0200 Subject: Remove part of if statement that should have been moved --- .../java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 70f49e60136..00aa5423087 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -199,7 +199,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder { private void addTlsClientAuthority(DeployState deployState, Element spec, ApplicationContainerCluster cluster) { var clientAuthorized = XML.getChild(spec, "client-authorize"); - if (clientAuthorized != null && deployState.tlsClientAuthority().isEmpty()) { + if (clientAuthorized != null) { if (deployState.tlsClientAuthority().isEmpty()) { throw new RuntimeException("client-authorize set, but security/clients.pem is missing"); } -- cgit v1.2.3