Merge pull request #10500 from vespa-engine/mortent/setup-tlsport-builder

Setup additional tls port 

7 files changed, 83 insertions, 89 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java
index 405cfb47c72..af9c452da11 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java
@@ -1,18 +1,12 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model.container;
 
-import com.yahoo.config.model.api.TlsSecrets;
 import com.yahoo.config.model.api.container.ContainerServiceType;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
 import com.yahoo.container.bundle.BundleInstantiationSpecification;
 import com.yahoo.osgi.provider.model.ComponentModel;
 import com.yahoo.prelude.fastsearch.FS4ResourcePool;
 import com.yahoo.vespa.model.container.component.Component;
-import com.yahoo.vespa.model.container.http.Http;
-import com.yahoo.vespa.model.container.http.JettyHttpServer;
-import com.yahoo.vespa.model.container.http.ssl.HostedSslConnectorFactory;
-
-import java.util.Optional;
 
 /**
  * A container that is typically used by container clusters set up from the user application.
@@ -25,27 +19,14 @@ public final class ApplicationContainer extends Container {
 
     private final boolean isHostedVespa;
 
-    public ApplicationContainer(AbstractConfigProducer parent, String name, int index, boolean isHostedVespa, Optional<TlsSecrets> tlsSecrets, Optional<String> tlsCa) {
-        this(parent, name, false, index, isHostedVespa, tlsSecrets, tlsCa);
+    public ApplicationContainer(AbstractConfigProducer parent, String name, int index, boolean isHostedVespa) {
+        this(parent, name, false, index, isHostedVespa);
     }
 
-    public ApplicationContainer(AbstractConfigProducer parent, String name, boolean retired, int index, boolean isHostedVespa, Optional<TlsSecrets> tlsSecrets, Optional<String> tlsCa) {
+    public ApplicationContainer(AbstractConfigProducer parent, String name, boolean retired, int index, boolean isHostedVespa) {
         super(parent, name, retired, index);
         this.isHostedVespa = isHostedVespa;
 
-        if (isHostedVespa && tlsSecrets.isPresent()) {
-
-            JettyHttpServer server = Optional.ofNullable(getHttp())
-                                             .map(Http::getHttpServer)
-                                             .orElse(getDefaultHttpServer());
-            if (server.getConnectorFactories().stream().noneMatch(connectorFactory -> connectorFactory instanceof HostedSslConnectorFactory)) {
-                String serverName = server.getComponentId().getName();
-                var connectorFactory = tlsCa
-                        .map(caCert -> new HostedSslConnectorFactory(serverName, tlsSecrets.get(), caCert))
-                        .orElseGet(() -> new HostedSslConnectorFactory(serverName, tlsSecrets.get()));
-                server.addConnector(connectorFactory);
-            }
-        }
         addComponent(getFS4ResourcePool()); // TODO Remove when FS4 based search protocol is gone
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
index eafb73d078f..414b127d6c4 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
@@ -56,7 +56,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
 
     private Optional<TlsSecrets> tlsSecrets;
     private Optional<String> tlsClientAuthority;
-    private boolean useTlsClientAuthority = false;
 
     private MbusParams mbusParams;
     private boolean messageBusEnabled = true;
@@ -205,10 +204,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
                         null))));
     }
 
-    public void useTlsClientAuthority(boolean value) {
-        this.useTlsClientAuthority = value;
-    }
-
     public static class MbusParams {
         // the amount of the maxpendingbytes to process concurrently, typically 0.2 (20%)
         final Double maxConcurrentFactor;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
index 00aa5423087..e47c00eeea3 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.model.container.xml;
 
 import com.google.common.collect.ImmutableList;
+import com.yahoo.component.ComponentId;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.Xml;
 import com.yahoo.config.application.api.ApplicationPackage;
@@ -57,7 +58,10 @@ import com.yahoo.vespa.model.container.component.Handler;
 import com.yahoo.vespa.model.container.component.chain.ProcessingHandler;
 import com.yahoo.vespa.model.container.docproc.ContainerDocproc;
 import com.yahoo.vespa.model.container.docproc.DocprocChains;
+import com.yahoo.vespa.model.container.http.ConnectorFactory;
 import com.yahoo.vespa.model.container.http.Http;
+import com.yahoo.vespa.model.container.http.JettyHttpServer;
+import com.yahoo.vespa.model.container.http.ssl.HostedSslConnectorFactory;
 import com.yahoo.vespa.model.container.http.xml.HttpBuilder;
 import com.yahoo.vespa.model.container.jersey.xml.RestApiBuilder;
 import com.yahoo.vespa.model.container.processing.ProcessingChains;
@@ -192,21 +196,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
         addClientProviders(deployState, spec, cluster);
         addServerProviders(deployState, spec, cluster);
 
-        addTlsClientAuthority(deployState, spec, cluster);
 
         addAthensCopperArgos(cluster, context);  // Must be added after nodes.
     }
 
-    private void addTlsClientAuthority(DeployState deployState, Element spec, ApplicationContainerCluster cluster) {
-        var clientAuthorized = XML.getChild(spec, "client-authorize");
-        if (clientAuthorized != null) {
-            if (deployState.tlsClientAuthority().isEmpty()) {
-                throw new RuntimeException("client-authorize set, but security/clients.pem is missing");
-            }
-            cluster.useTlsClientAuthority(true);
-        }
-    }
-
     private void addSecretStore(ApplicationContainerCluster cluster, Element spec) {
         Element secretStoreElement = XML.getChild(spec, "secret-store");
         if (secretStoreElement != null) {
@@ -335,6 +328,33 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
         if (httpElement != null) {
             cluster.setHttp(buildHttp(deployState, cluster, httpElement));
         }
+
+        // If the deployment contains certificate/private key reference, setup TLS port
+        if (deployState.tlsSecrets().isPresent()) {
+            boolean authorizeClient = XML.getChild(spec, "client-authorize") != null;
+            if (authorizeClient) {
+                if (deployState.tlsClientAuthority().isEmpty()) {
+                    throw new RuntimeException("client-authorize set, but security/clients.pem is missing");
+                }
+            }
+
+            if(httpElement == null) {
+                cluster.setHttp(new Http(Collections.emptyList()));
+            }
+            if(cluster.getHttp().getHttpServer() == null) {
+                JettyHttpServer defaultHttpServer = new JettyHttpServer(new ComponentId("DefaultHttpServer"));
+                cluster.getHttp().setHttpServer(defaultHttpServer);
+                defaultHttpServer.addConnector(new ConnectorFactory("SearchServer", Defaults.getDefaults().vespaWebServicePort()));
+
+            }
+            JettyHttpServer server = cluster.getHttp().getHttpServer();
+
+            String serverName = server.getComponentId().getName();
+            HostedSslConnectorFactory connectorFactory = authorizeClient
+                    ? new HostedSslConnectorFactory(serverName, deployState.tlsSecrets().get(), deployState.tlsClientAuthority().get())
+                    : new HostedSslConnectorFactory(serverName, deployState.tlsSecrets().get());
+            server.addConnector(connectorFactory);
+        }
     }
 
     private Http buildHttp(DeployState deployState, ApplicationContainerCluster cluster, Element httpElement) {
@@ -452,7 +472,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
     }
 
     private void addStandaloneNode(ApplicationContainerCluster cluster) {
-        ApplicationContainer container =  new ApplicationContainer(cluster, "standalone", cluster.getContainers().size(), cluster.isHostedVespa(), cluster.getTlsSecrets(), cluster.getTlsClientAuthority());
+        ApplicationContainer container =  new ApplicationContainer(cluster, "standalone", cluster.getContainers().size(), cluster.isHostedVespa());
         cluster.addContainers(Collections.singleton(container));
     }
 
@@ -518,7 +538,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
         Element nodesElement = XML.getChild(containerElement, "nodes");
         Element rotationsElement = XML.getChild(containerElement, "rotations");
         if (nodesElement == null) { // default single node on localhost
-            ApplicationContainer node = new ApplicationContainer(cluster, "container.0", 0, cluster.isHostedVespa(), cluster.getTlsSecrets(), cluster.getTlsClientAuthority());
+            ApplicationContainer node = new ApplicationContainer(cluster, "container.0", 0, cluster.isHostedVespa());
             HostResource host = allocateSingleNodeHost(cluster, log, containerElement, context);
             node.setHostResource(host);
             node.initService(context.getDeployLogger());
@@ -707,7 +727,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
         List<ApplicationContainer> nodes = new ArrayList<>();
         for (Map.Entry<HostResource, ClusterMembership> entry : hosts.entrySet()) {
             String id = "container." + entry.getValue().index();
-            ApplicationContainer container = new ApplicationContainer(cluster, id, entry.getValue().retired(), entry.getValue().index(), cluster.isHostedVespa(), cluster.getTlsSecrets(), cluster.getTlsClientAuthority());
+            ApplicationContainer container = new ApplicationContainer(cluster, id, entry.getValue().retired(), entry.getValue().index(), cluster.isHostedVespa());
             container.setHostResource(entry.getKey());
             container.initService(deployLogger);
             nodes.add(container);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java
index c976a7fb153..4d9ba2f920e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java
@@ -26,9 +26,7 @@ public class ContainerServiceBuilder extends VespaDomBuilder.DomConfigProducerBu
                 parent,
                 id,
                 index,
-                deployState.isHosted(),
-                deployState.tlsSecrets(),
-                deployState.tlsClientAuthority()
+                deployState.isHosted()
         );
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java
index f028f0ac0cc..74caf2d8026 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java
@@ -324,7 +324,7 @@ public class Content extends ConfigModel {
                 if (!processedHosts.contains(host)) {
                     String containerName = String.valueOf(searchNode.getDistributionKey());
                     ApplicationContainer docprocService = new ApplicationContainer(indexingCluster, containerName, index,
-                                                                                   modelContext.getDeployState().isHosted(), modelContext.getDeployState().tlsSecrets(), Optional.empty());
+                                                                                   modelContext.getDeployState().isHosted());
                     index++;
                     docprocService.useDynamicPorts();
                     docprocService.setHostResource(host);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
index 89169c44079..ec8a810c1a7 100755
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
@@ -5,7 +5,6 @@ import com.yahoo.cloud.config.ClusterInfoConfig;
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.cloud.config.RoutingProviderConfig;
 import com.yahoo.config.application.api.DeployLogger;
-import com.yahoo.config.model.api.TlsSecrets;
 import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.model.deploy.TestProperties;
 import com.yahoo.config.model.test.MockRoot;
@@ -14,7 +13,6 @@ import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.container.handler.ThreadpoolConfig;
-import com.yahoo.jdisc.http.ConnectorConfig;
 import com.yahoo.search.config.QrStartConfig;
 import com.yahoo.vespa.model.Host;
 import com.yahoo.vespa.model.HostResource;
@@ -22,22 +20,15 @@ import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer;
 import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainerCluster;
 import com.yahoo.vespa.model.container.component.Component;
 import com.yahoo.vespa.model.container.docproc.ContainerDocproc;
-import com.yahoo.vespa.model.container.http.ConnectorFactory;
 import com.yahoo.vespa.model.container.search.ContainerSearch;
 import com.yahoo.vespa.model.container.search.searchchain.SearchChains;
-import org.hamcrest.Matchers;
 import org.junit.Test;
 
 import java.util.Collection;
 import java.util.Iterator;
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Collectors;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
 
 /**
  * @author Simon Thoresen Hult
@@ -219,40 +210,8 @@ public class ContainerClusterTest {
         assertEquals(0, cluster.getAllComponents().stream().map(c -> c.getClassId().getName()).filter(c -> c.equals("com.yahoo.jdisc.http.filter.security.RoutingConfigProvider")).count());
     }
 
-    @Test
-    public void requireThatProvidingTlsSecretOpensPort4443() {
-        DeployState state = new DeployState.Builder().properties(new TestProperties().setHostedVespa(true).setTlsSecrets(Optional.of(new TlsSecrets("CERT", "KEY")))).build();
-        MockRoot root = new MockRoot("foo", state);
-        ApplicationContainerCluster cluster = new ApplicationContainerCluster(root, "container0", "container1", state);
-
-        addContainer(state.getDeployLogger(), cluster, "c1", "host-c1");
-        Optional<ApplicationContainer> container = cluster.getContainers().stream().findFirst();
-        assertTrue(container.isPresent());
-
-        var httpServer = (container.get().getHttp() == null) ? container.get().getDefaultHttpServer() : container.get().getHttp().getHttpServer();
-
-        // Verify that there are two connectors
-        List<ConnectorFactory> connectorFactories = httpServer.getConnectorFactories();
-        assertEquals(2, connectorFactories.size());
-        List<Integer> ports = connectorFactories.stream()
-                                                .map(ConnectorFactory::getListenPort)
-                                                .collect(Collectors.toList());
-        assertThat(ports, Matchers.containsInAnyOrder(8080, 4443));
-
-        ConnectorFactory tlsPort = connectorFactories.stream().filter(connectorFactory -> connectorFactory.getListenPort() == 4443).findFirst().orElseThrow();
-
-        ConnectorConfig.Builder builder = new ConnectorConfig.Builder();
-        tlsPort.getConfig(builder);
-
-        ConnectorConfig connectorConfig = new ConnectorConfig(builder);
-        assertTrue(connectorConfig.ssl().enabled());
-        assertEquals("CERT", connectorConfig.ssl().certificate());
-        assertEquals("KEY", connectorConfig.ssl().privateKey());
-        assertEquals(4443, connectorConfig.listenPort());
-    }
-
     private static void addContainer(DeployLogger deployLogger, ApplicationContainerCluster cluster, String name, String hostName) {
-        ApplicationContainer container = new ApplicationContainer(cluster, name, 0, cluster.isHostedVespa(), cluster.getTlsSecrets(), cluster.getTlsClientAuthority());
+        ApplicationContainer container = new ApplicationContainer(cluster, name, 0, cluster.isHostedVespa());
         container.setHostResource(new HostResource(new Host(null, hostName)));
         container.initService(deployLogger);
         cluster.addContainer(container);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
index 58947de398a..d2a9b8172b3 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
@@ -5,6 +5,7 @@ import com.yahoo.component.ComponentId;
 import com.yahoo.config.application.api.ApplicationPackage;
 import com.yahoo.config.model.NullConfigModelRegistry;
 import com.yahoo.config.model.api.ContainerEndpoint;
+import com.yahoo.config.model.api.TlsSecrets;
 import com.yahoo.config.model.builder.xml.test.DomBuilderTest;
 import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.model.deploy.TestProperties;
@@ -23,6 +24,7 @@ import com.yahoo.container.handler.observability.ApplicationStatusHandler;
 import com.yahoo.container.jdisc.JdiscBindingsConfig;
 import com.yahoo.container.servlet.ServletConfigConfig;
 import com.yahoo.container.usability.BindingsOverviewHandler;
+import com.yahoo.jdisc.http.ConnectorConfig;
 import com.yahoo.jdisc.http.ServletPathsConfig;
 import com.yahoo.net.HostName;
 import com.yahoo.path.Path;
@@ -30,19 +32,21 @@ import com.yahoo.prelude.cluster.QrMonitorConfig;
 import com.yahoo.search.config.QrStartConfig;
 import com.yahoo.vespa.model.AbstractService;
 import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.container.ApplicationContainer;
 import com.yahoo.vespa.model.container.Container;
 import com.yahoo.vespa.model.container.ContainerCluster;
 import com.yahoo.vespa.model.container.SecretStore;
 import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.container.http.ConnectorFactory;
 import com.yahoo.vespa.model.content.utils.ContentClusterUtils;
 import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg;
+import org.hamcrest.Matchers;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 import org.w3c.dom.Element;
 import org.xml.sax.SAXException;
 
-import java.io.File;
 import java.io.IOException;
 import java.io.StringReader;
 import java.util.List;
@@ -659,7 +663,11 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase {
                 "  <client-authorize />",
                 "</container>");
         try {
-            createModel(root, clusterElem);
+            DeployState state = new DeployState.Builder().properties(
+                    new TestProperties()
+                            .setHostedVespa(true)
+                            .setTlsSecrets(Optional.of(new TlsSecrets("CERT", "KEY")))).build();
+            createModel(root, state, null, clusterElem);
         } catch (RuntimeException e) {
             assertEquals(e.getMessage(), "client-authorize set, but security/clients.pem is missing");
             return;
@@ -706,6 +714,39 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase {
         assertEquals("KMP_SETTING=1 KMP_AFFINITY=granularity=fine,verbose,compact,1,0 ", qrStartConfig.qrs().env());
     }
 
+    @Test
+    public void requireThatProvidingTlsSecretOpensPort4443() {
+        Element clusterElem = DomBuilderTest.parse(
+                "<container version='1.0'>",
+                nodesXml,
+                "</container>" );
+
+        DeployState state = new DeployState.Builder().properties(new TestProperties().setHostedVespa(true).setTlsSecrets(Optional.of(new TlsSecrets("CERT", "KEY")))).build();
+        createModel(root, state, null, clusterElem);
+        ApplicationContainer container = (ApplicationContainer)root.getProducer("container/container.0");
+
+        // Verify that there are two connectors
+        List<ConnectorFactory> connectorFactories = container.getHttp().getHttpServer().getConnectorFactories();
+        assertEquals(2, connectorFactories.size());
+        List<Integer> ports = connectorFactories.stream()
+                .map(ConnectorFactory::getListenPort)
+                .collect(Collectors.toList());
+        assertThat(ports, Matchers.containsInAnyOrder(8080, 4443));
+
+        ConnectorFactory tlsPort = connectorFactories.stream().filter(connectorFactory -> connectorFactory.getListenPort() == 4443).findFirst().orElseThrow();
+
+        ConnectorConfig.Builder builder = new ConnectorConfig.Builder();
+        tlsPort.getConfig(builder);
+
+
+        ConnectorConfig connectorConfig = new ConnectorConfig(builder);
+        assertTrue(connectorConfig.ssl().enabled());
+        assertEquals("CERT", connectorConfig.ssl().certificate());
+        assertEquals("KEY", connectorConfig.ssl().privateKey());
+        assertEquals(4443, connectorConfig.listenPort());
+    }
+
+
     private Element generateContainerElementWithRenderer(String rendererId) {
         return DomBuilderTest.parse(
                 "<container id='default' version='1.0'>",