diff options
65 files changed, 703 insertions, 279 deletions
diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 9f86fe4dea2..8d83d16a272 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -111,6 +111,7 @@ "public java.util.List getQueryProfileFiles()", "public java.util.List getQueryProfileTypeFiles()", "public java.util.List getPageTemplateFiles()", + "public com.yahoo.config.application.api.ApplicationFile getClientSecurityFile()", "public abstract java.lang.String getHostSource()", "public abstract java.lang.String getServicesSource()", "public abstract java.util.Optional getDeployment()", @@ -148,6 +149,7 @@ "public static final com.yahoo.path.Path RULES_DIR", "public static final com.yahoo.path.Path DEPLOYMENT_FILE", "public static final com.yahoo.path.Path VALIDATION_OVERRIDES", + "public static final com.yahoo.path.Path SECURITY_DIR", "public static final java.lang.String SD_NAME_SUFFIX", "public static final java.lang.String RANKEXPRESSION_NAME_SUFFIX", "public static final java.lang.String RULES_NAME_SUFFIX", diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index 06f8034453d..5cd119dcf65 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -27,7 +27,6 @@ import java.util.Map; import java.util.Optional; import java.util.jar.JarEntry; import java.util.jar.JarFile; -import java.util.stream.Collectors; /** * Represents an application package, that is, used as input when creating a VespaModel and as @@ -73,6 +72,8 @@ public interface ApplicationPackage { Path DEPLOYMENT_FILE = Path.fromString("deployment.xml"); Path VALIDATION_OVERRIDES = Path.fromString("validation-overrides.xml"); + Path SECURITY_DIR = Path.fromString("security"); + String SD_NAME_SUFFIX = ".sd"; String RANKEXPRESSION_NAME_SUFFIX = ".expression"; String RULES_NAME_SUFFIX = ".sr"; @@ -178,6 +179,9 @@ public interface ApplicationPackage { /** Does {@link #getFiles} on the page template directory and gets all xml files */ default List<NamedReader> getPageTemplateFiles() { return getFiles(PAGE_TEMPLATES_DIR,".xml"); } + /** Returns handle for the file containing client certificate authorities */ + default ApplicationFile getClientSecurityFile() { return getFile(SECURITY_DIR.append("clients.pem")); } + //For generating error messages String getHostSource(); String getServicesSource(); diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java b/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java index 3cb4cedcbac..6a8b5a237ab 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java @@ -1,30 +1,30 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.api; - public class TlsSecrets { +public class TlsSecrets { public static final TlsSecrets MISSING = new TlsSecrets(); - private final String certificate; + private final String certificate; private final String key; - private TlsSecrets() { - this(null,null); + private TlsSecrets() { + this(null, null); } - public TlsSecrets(String certificate, String key) { + public TlsSecrets(String certificate, String key) { this.certificate = certificate; this.key = key; } - public String certificate() { + public String certificate() { return certificate; } - public String key() { + public String key() { return key; } - public boolean isMissing() { + public boolean isMissing() { return this == MISSING; } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 86c20bf96af..e8e12888768 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -24,6 +24,7 @@ import com.yahoo.config.model.provision.SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.Zone; +import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.SearchBuilder; @@ -39,8 +40,10 @@ import com.yahoo.vespa.model.container.search.SemanticRules; import com.yahoo.vespa.model.search.SearchDefinition; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.Reader; +import java.io.UncheckedIOException; import java.time.Instant; import java.util.Collection; import java.util.Collections; @@ -263,6 +266,22 @@ public class DeployState implements ConfigDefinitionStore { public Optional<TlsSecrets> tlsSecrets() { return properties.tlsSecrets(); } + public Optional<String> tlsClientAuthority() { + var caFile = applicationPackage.getClientSecurityFile(); + if (caFile.exists()) { + try { + var caPem = IOUtils.readAll(caFile.createReader()); + return Optional.of(caPem); + } catch (FileNotFoundException e) { + return Optional.empty(); + } catch (IOException e) { + throw new UncheckedIOException("Failed reading certificate from application: " + caFile.getPath(), e); + } + } else { + return Optional.empty(); + } + } + public static class Builder { private ApplicationPackage applicationPackage = MockApplicationPackage.createEmpty(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index d27812a80dd..82f0c87d074 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -26,11 +26,11 @@ public final class ApplicationContainer extends Container { private final boolean isHostedVespa; - public ApplicationContainer(AbstractConfigProducer parent, String name, int index, boolean isHostedVespa, Optional<TlsSecrets> tlsSecrets) { - this(parent, name, false, index, isHostedVespa, tlsSecrets); + public ApplicationContainer(AbstractConfigProducer parent, String name, int index, boolean isHostedVespa, Optional<TlsSecrets> tlsSecrets, Optional<String> tlsCa) { + this(parent, name, false, index, isHostedVespa, tlsSecrets, tlsCa); } - public ApplicationContainer(AbstractConfigProducer parent, String name, boolean retired, int index, boolean isHostedVespa, Optional<TlsSecrets> tlsSecrets) { + public ApplicationContainer(AbstractConfigProducer parent, String name, boolean retired, int index, boolean isHostedVespa, Optional<TlsSecrets> tlsSecrets, Optional<String> tlsCa) { super(parent, name, retired, index); this.isHostedVespa = isHostedVespa; @@ -40,8 +40,17 @@ public final class ApplicationContainer extends Container { JettyHttpServer server = Optional.ofNullable(getHttp()) .map(Http::getHttpServer) .orElse(getDefaultHttpServer()); - server.addConnector(new ConnectorFactory(connectorName, 4443, - new ConfiguredDirectSslProvider(server.getComponentId().getName(), tlsSecrets.get().key(), tlsSecrets.get().certificate(), null, null))); + + var sslProvider = new ConfiguredDirectSslProvider( + server.getComponentId().getName(), + tlsSecrets.get().key(), + tlsSecrets.get().certificate(), + null, + tlsCa.orElse(null), + null + ); + + server.addConnector(new ConnectorFactory(connectorName, 4443, sslProvider)); } addComponent(getFS4ResourcePool()); // TODO Remove when FS4 based search protocol is gone } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 473971c5e7a..2b9cd18d64b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -56,6 +56,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private ContainerModelEvaluation modelEvaluation; private Optional<TlsSecrets> tlsSecrets; + private Optional<String> tlsClientAuthority; + private boolean useTlsClientAuthority = false; private final boolean enableGroupingSessionCache; private MbusParams mbusParams; @@ -65,6 +67,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat super(parent, subId, name, deployState); this.tlsSecrets = deployState.tlsSecrets(); + this.tlsClientAuthority = deployState.tlsClientAuthority(); this.enableGroupingSessionCache = deployState.getProperties().enableGroupingSessionCache(); restApiGroup = new ConfigProducerGroup<>(this, "rest-api"); servletGroup = new ConfigProducerGroup<>(this, "servlet"); @@ -183,6 +186,10 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat return tlsSecrets; } + public Optional<String> getTlsClientAuthority() { + return tlsClientAuthority; + } + public boolean enableGroupingSessionCache() { return enableGroupingSessionCache; } @@ -205,6 +212,10 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat null)))); } + public void useTlsClientAuthority(boolean value) { + this.useTlsClientAuthority = value; + } + public static class MbusParams { // the amount of the maxpendingbytes to process concurrently, typically 0.2 (20%) final Double maxConcurrentFactor; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredDirectSslProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredDirectSslProvider.java index 28dba3331d3..b47aa501ece 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredDirectSslProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredDirectSslProvider.java @@ -26,9 +26,10 @@ public class ConfiguredDirectSslProvider extends SimpleComponent implements Conn private final String privateKey; private final String certificate; private final String caCertificatePath; + private final String caCertificate; private final ConnectorConfig.Ssl.ClientAuth.Enum clientAuthentication; - public ConfiguredDirectSslProvider(String servername, String privateKey, String certificate, String caCertificatePath, String clientAuthentication) { + public ConfiguredDirectSslProvider(String servername, String privateKey, String certificate, String caCertificatePath, String caCertificate, String clientAuthentication) { super(new ComponentModel( new BundleInstantiationSpecification(new ComponentId(COMPONENT_ID_PREFIX+servername), fromString(COMPONENT_CLASS), @@ -36,6 +37,7 @@ public class ConfiguredDirectSslProvider extends SimpleComponent implements Conn this.privateKey = privateKey; this.certificate = certificate; this.caCertificatePath = caCertificatePath; + this.caCertificate = caCertificate; this.clientAuthentication = mapToConfigEnum(clientAuthentication); } @@ -45,6 +47,7 @@ public class ConfiguredDirectSslProvider extends SimpleComponent implements Conn builder.ssl.privateKey(privateKey); builder.ssl.certificate(certificate); builder.ssl.caCertificateFile(Optional.ofNullable(caCertificatePath).orElse("")); + builder.ssl.caCertificate(Optional.ofNullable(caCertificate).orElse("")); builder.ssl.clientAuth(clientAuthentication); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 2bfb1da9dcb..00aa5423087 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -192,9 +192,21 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addClientProviders(deployState, spec, cluster); addServerProviders(deployState, spec, cluster); + addTlsClientAuthority(deployState, spec, cluster); + addAthensCopperArgos(cluster, context); // Must be added after nodes. } + private void addTlsClientAuthority(DeployState deployState, Element spec, ApplicationContainerCluster cluster) { + var clientAuthorized = XML.getChild(spec, "client-authorize"); + if (clientAuthorized != null) { + if (deployState.tlsClientAuthority().isEmpty()) { + throw new RuntimeException("client-authorize set, but security/clients.pem is missing"); + } + cluster.useTlsClientAuthority(true); + } + } + private void addSecretStore(ApplicationContainerCluster cluster, Element spec) { Element secretStoreElement = XML.getChild(spec, "secret-store"); if (secretStoreElement != null) { @@ -440,7 +452,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } private void addStandaloneNode(ApplicationContainerCluster cluster) { - ApplicationContainer container = new ApplicationContainer(cluster, "standalone", cluster.getContainers().size(), cluster.isHostedVespa(), cluster.getTlsSecrets()); + ApplicationContainer container = new ApplicationContainer(cluster, "standalone", cluster.getContainers().size(), cluster.isHostedVespa(), cluster.getTlsSecrets(), cluster.getTlsClientAuthority()); cluster.addContainers(Collections.singleton(container)); } @@ -506,7 +518,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Element nodesElement = XML.getChild(containerElement, "nodes"); Element rotationsElement = XML.getChild(containerElement, "rotations"); if (nodesElement == null) { // default single node on localhost - ApplicationContainer node = new ApplicationContainer(cluster, "container.0", 0, cluster.isHostedVespa(), cluster.getTlsSecrets()); + ApplicationContainer node = new ApplicationContainer(cluster, "container.0", 0, cluster.isHostedVespa(), cluster.getTlsSecrets(), cluster.getTlsClientAuthority()); HostResource host = allocateSingleNodeHost(cluster, log, containerElement, context); node.setHostResource(host); node.initService(context.getDeployLogger()); @@ -695,7 +707,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { List<ApplicationContainer> nodes = new ArrayList<>(); for (Map.Entry<HostResource, ClusterMembership> entry : hosts.entrySet()) { String id = "container." + entry.getValue().index(); - ApplicationContainer container = new ApplicationContainer(cluster, id, entry.getValue().retired(), entry.getValue().index(), cluster.isHostedVespa(), cluster.getTlsSecrets()); + ApplicationContainer container = new ApplicationContainer(cluster, id, entry.getValue().retired(), entry.getValue().index(), cluster.isHostedVespa(), cluster.getTlsSecrets(), cluster.getTlsClientAuthority()); container.setHostResource(entry.getKey()); container.initService(deployLogger); nodes.add(container); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java index 46271d3c0a2..c976a7fb153 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java @@ -22,7 +22,14 @@ public class ContainerServiceBuilder extends VespaDomBuilder.DomConfigProducerBu @Override protected ApplicationContainer doBuild(DeployState deployState, AbstractConfigProducer parent, Element nodeElem) { - return new ApplicationContainer(parent, id, index, deployState.isHosted(), deployState.tlsSecrets()); + return new ApplicationContainer( + parent, + id, + index, + deployState.isHosted(), + deployState.tlsSecrets(), + deployState.tlsClientAuthority() + ); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index 8eda707be99..f028f0ac0cc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -324,7 +324,7 @@ public class Content extends ConfigModel { if (!processedHosts.contains(host)) { String containerName = String.valueOf(searchNode.getDistributionKey()); ApplicationContainer docprocService = new ApplicationContainer(indexingCluster, containerName, index, - modelContext.getDeployState().isHosted(), modelContext.getDeployState().tlsSecrets()); + modelContext.getDeployState().isHosted(), modelContext.getDeployState().tlsSecrets(), Optional.empty()); index++; docprocService.useDynamicPorts(); docprocService.setHostResource(host); diff --git a/config-model/src/test/cfg/admin/metricconfig/searchdefinitions/music.sd b/config-model/src/test/cfg/admin/metricconfig/searchdefinitions/music.sd index f7b182f66ea..32e8451d8e2 100644 --- a/config-model/src/test/cfg/admin/metricconfig/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/admin/metricconfig/searchdefinitions/music.sd @@ -4,12 +4,10 @@ search music { field f1 type string { indexing: summary | index # index-to: f1, all - header } field f2 type string { indexing: summary | index # index-to: f2, all - body } } } diff --git a/config-model/src/test/cfg/application/app1/searchdefinitions/music.sd b/config-model/src/test/cfg/application/app1/searchdefinitions/music.sd index dda729d6a35..693afbd308d 100644 --- a/config-model/src/test/cfg/application/app1/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/application/app1/searchdefinitions/music.sd @@ -9,35 +9,28 @@ search music { indexing: summary | index # How this field should be indexed # index-to: title, default # Create two indexes rank-type: about # Type of ranking settings to apply - header } field artist type string { indexing: summary | attribute | index # index-to: artist, default rank-type:about - header } field year type int { indexing: summary | attribute - header } # Increase rank score of popular documents regardless of query field popularity type int { indexing: summary | attribute - body } field url type uri { indexing: summary | index - header } - field cover type raw { - body - } + field cover type raw {} } diff --git a/config-model/src/test/cfg/application/app_genericservices/searchdefinitions/music.sd b/config-model/src/test/cfg/application/app_genericservices/searchdefinitions/music.sd index dda729d6a35..693afbd308d 100644 --- a/config-model/src/test/cfg/application/app_genericservices/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/application/app_genericservices/searchdefinitions/music.sd @@ -9,35 +9,28 @@ search music { indexing: summary | index # How this field should be indexed # index-to: title, default # Create two indexes rank-type: about # Type of ranking settings to apply - header } field artist type string { indexing: summary | attribute | index # index-to: artist, default rank-type:about - header } field year type int { indexing: summary | attribute - header } # Increase rank score of popular documents regardless of query field popularity type int { indexing: summary | attribute - body } field url type uri { indexing: summary | index - header } - field cover type raw { - body - } + field cover type raw {} } diff --git a/config-model/src/test/cfg/routing/content_two_clusters/searchdefinitions/mobile.sd b/config-model/src/test/cfg/routing/content_two_clusters/searchdefinitions/mobile.sd index feae86fe966..3abc9238e8a 100644 --- a/config-model/src/test/cfg/routing/content_two_clusters/searchdefinitions/mobile.sd +++ b/config-model/src/test/cfg/routing/content_two_clusters/searchdefinitions/mobile.sd @@ -4,12 +4,10 @@ search mobile { field f1 type string { indexing: summary | index # index-to: f1, all - header } field f2 type string { indexing: summary | index # index-to: f2, all - body } } } diff --git a/config-model/src/test/cfg/search/data/travel/searchdefinitions/TTEdge.sd b/config-model/src/test/cfg/search/data/travel/searchdefinitions/TTEdge.sd index 8cc6d05584c..e6707345235 100644 --- a/config-model/src/test/cfg/search/data/travel/searchdefinitions/TTEdge.sd +++ b/config-model/src/test/cfg/search/data/travel/searchdefinitions/TTEdge.sd @@ -4,7 +4,6 @@ document TTEdge { # This field will contain a colon separate map for travel times per transport mode field TransportMode type array<string> { indexing: summary | index - header } } diff --git a/config-model/src/test/cfg/search/data/travel/searchdefinitions/TTPOI.sd b/config-model/src/test/cfg/search/data/travel/searchdefinitions/TTPOI.sd index 0cb0596b31c..c39ef03add5 100644 --- a/config-model/src/test/cfg/search/data/travel/searchdefinitions/TTPOI.sd +++ b/config-model/src/test/cfg/search/data/travel/searchdefinitions/TTPOI.sd @@ -5,14 +5,12 @@ document TTPOI { field Categories type array<string> { indexing: summary | index # index-to: Categories - header } # sub catagories associated with the POI field SubCategories type array<string> { indexing: summary | index # index-to: SubCategories - header } } diff --git a/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/searchdefinitions/music.sd b/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/searchdefinitions/music.sd index 0d3859b65a0..da9e2d14c55 100644 --- a/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/searchdefinitions/music.sd @@ -4,12 +4,10 @@ search music { field f1 type string { indexing: summary | index # index-to: f1, all - header } field f2 type string { indexing: summary | index # index-to: f2, all - body } } } diff --git a/config-model/src/test/cfg/storage/app_index_higher_than_num_nodes/searchdefinitions/music.sd b/config-model/src/test/cfg/storage/app_index_higher_than_num_nodes/searchdefinitions/music.sd index 0d3859b65a0..da9e2d14c55 100644 --- a/config-model/src/test/cfg/storage/app_index_higher_than_num_nodes/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/storage/app_index_higher_than_num_nodes/searchdefinitions/music.sd @@ -4,12 +4,10 @@ search music { field f1 type string { indexing: summary | index # index-to: f1, all - header } field f2 type string { indexing: summary | index # index-to: f2, all - body } } } diff --git a/config-model/src/test/cfg/storage/clustercontroller_advanced/searchdefinitions/music.sd b/config-model/src/test/cfg/storage/clustercontroller_advanced/searchdefinitions/music.sd index 0d3859b65a0..da9e2d14c55 100644 --- a/config-model/src/test/cfg/storage/clustercontroller_advanced/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/storage/clustercontroller_advanced/searchdefinitions/music.sd @@ -4,12 +4,10 @@ search music { field f1 type string { indexing: summary | index # index-to: f1, all - header } field f2 type string { indexing: summary | index # index-to: f2, all - body } } } diff --git a/config-model/src/test/configmodel/types/types.sd b/config-model/src/test/configmodel/types/types.sd index f34a6776b11..9bd9602008c 100644 --- a/config-model/src/test/configmodel/types/types.sd +++ b/config-model/src/test/configmodel/types/types.sd @@ -90,9 +90,9 @@ search types { #field wildcardfield2 type map<?,?> { #} - field arrarr type array<array<array<string>>> {header} - field maparr type array<map<string, string>> {header} - field complexarray type array< map<int, array<array<string>>> > {body} + field arrarr type array<array<array<string>>> {} + field maparr type array<map<string, string>> {} + field complexarray type array< map<int, array<array<string>>> > {} struct mystruct { field bytearr type array<byte>{} @@ -101,9 +101,9 @@ search types { field structfield type string {} } - field mystructfield type mystruct {header} - field mystructmap type map<int, mystruct> {header} - field mystructarr type array<mystruct> {header} + field mystructfield type mystruct {} + field mystructmap type map<int, mystruct> {} + field mystructarr type array<mystruct> {} struct folder { field Version type int {} @@ -130,7 +130,6 @@ search types { create-if-nonexistent remove-if-zero } - header } # Field defined same way as tag @@ -140,7 +139,6 @@ search types { create-if-nonexistent remove-if-zero } - header } } diff --git a/config-model/src/test/derived/inheritstruct/child.sd b/config-model/src/test/derived/inheritstruct/child.sd index cd3d4f51458..5ac69c429e1 100644 --- a/config-model/src/test/derived/inheritstruct/child.sd +++ b/config-model/src/test/derived/inheritstruct/child.sd @@ -3,7 +3,6 @@ search child { document child inherits parent { field child_struct_field type my_struct { indexing: summary | index - header match: prefix } } diff --git a/config-model/src/test/derived/mail/mail.sd b/config-model/src/test/derived/mail/mail.sd index 6d30891f307..6c2c51eaa8a 100644 --- a/config-model/src/test/derived/mail/mail.sd +++ b/config-model/src/test/derived/mail/mail.sd @@ -54,39 +54,31 @@ search mail { field body type string { indexing: summary | index match: substring - body } field attachmentcount type int { indexing: summary | index - body } field attachmentnames type string { indexing: index - body } field attachmenttypes type string { indexing: index - body } field attachmentlanguages type string { indexing: index match: prefix - body } field attachmentcontent type string { indexing: summary | index match: prefix - body } - field attachments type raw[] { - body - } + field attachments type raw[] {} } diff --git a/config-model/src/test/derived/music3/music3.sd b/config-model/src/test/derived/music3/music3.sd index 8aeed27d29f..a9da946f8c1 100644 --- a/config-model/src/test/derived/music3/music3.sd +++ b/config-model/src/test/derived/music3/music3.sd @@ -7,35 +7,25 @@ search music3 { indexing: summary | index # index-to: title, default rank-type: about - - header } field artist type string { indexing: summary | attribute | index # index-to: artist, default rank-type:about - - header } field year type int { indexing: summary | attribute - - header } # Increase rank score of popular documents regardless of query field popularity type int { indexing: summary | attribute - - header } field url type uri { indexing: summary | index - - header } } diff --git a/config-model/src/test/derived/streamingjuniper/streamingjuniper.sd b/config-model/src/test/derived/streamingjuniper/streamingjuniper.sd index 0d81ecd21f3..92b833ffd5d 100644 --- a/config-model/src/test/derived/streamingjuniper/streamingjuniper.sd +++ b/config-model/src/test/derived/streamingjuniper/streamingjuniper.sd @@ -3,12 +3,10 @@ search streamingjuniper { document streamingjuniper { field f1 type string { indexing: index | summary - header bolding: on } field f2 type string { indexing: index | summary - header summary: dynamic } } diff --git a/config-model/src/test/derived/streamingstructdefault/streamingstructdefault.sd b/config-model/src/test/derived/streamingstructdefault/streamingstructdefault.sd index 6d16a1b3808..46112db7454 100644 --- a/config-model/src/test/derived/streamingstructdefault/streamingstructdefault.sd +++ b/config-model/src/test/derived/streamingstructdefault/streamingstructdefault.sd @@ -8,11 +8,9 @@ search streamingstructdefault { field f1 type array<string> { indexing: index | summary summary-to: default - header } field f2 type array<sct> { indexing: index | summary - header } } document-summary default { diff --git a/config-model/src/test/derived/types/types.sd b/config-model/src/test/derived/types/types.sd index 839cb08dbd6..37ac6e7ee11 100644 --- a/config-model/src/test/derived/types/types.sd +++ b/config-model/src/test/derived/types/types.sd @@ -93,9 +93,9 @@ search types { #field wildcardfield2 type map<?,?> { #} - field arrarr type array<array<array<string>>> {header} - field maparr type array<map<string, string>> {header} - field complexarray type array< map<int, array<array<string>>> > {body} + field arrarr type array<array<array<string>>> {} + field maparr type array<map<string, string>> {} + field complexarray type array< map<int, array<array<string>>> > {} struct mystruct { field bytearr type array<byte>{} @@ -104,9 +104,9 @@ search types { field structfield type string {} } - field mystructfield type mystruct {header} - field mystructmap type map<int, mystruct> {header} - field mystructarr type array<mystruct> {header} + field mystructfield type mystruct {} + field mystructmap type map<int, mystruct> {} + field mystructarr type array<mystruct> {} struct folder { field Version type int {} @@ -133,7 +133,6 @@ search types { create-if-nonexistent remove-if-zero } - header } # Field defined same way as tag @@ -143,14 +142,11 @@ search types { create-if-nonexistent remove-if-zero } - header } } - field pst_sta_boldingoff_nomatch_tag_01 type tag { - body - } + field pst_sta_boldingoff_nomatch_tag_01 type tag {} field other type long { indexing: input along | attribute diff --git a/config-model/src/test/examples/header_body.sd b/config-model/src/test/examples/header_body.sd new file mode 100644 index 00000000000..561ec47b067 --- /dev/null +++ b/config-model/src/test/examples/header_body.sd @@ -0,0 +1,18 @@ +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Search definition with (ignored) header and body statements +# TODO: Remove in Vespa 8 +search header_body { + + document header_body { + + field title type string { + indexing: summary | index + header + } + + field description type string { + indexing: summary | index + body + } + } +} diff --git a/config-model/src/test/examples/rankpropvars.sd b/config-model/src/test/examples/rankpropvars.sd index 28959edbc09..bac02ea8316 100644 --- a/config-model/src/test/examples/rankpropvars.sd +++ b/config-model/src/test/examples/rankpropvars.sd @@ -52,28 +52,22 @@ document music { field title type string { indexing: index | summary - body } field artist type string { ## index-to: a indexing: index | summary - body } field year type int { indexing: attribute | summary ## index-to: y - body } - field url type uri { - body - } + field url type uri {} field Popularity type string { indexing: attribute | summary - body } } diff --git a/config-model/src/test/examples/simple.sd b/config-model/src/test/examples/simple.sd index 0435ea439df..8a97db6afa2 100644 --- a/config-model/src/test/examples/simple.sd +++ b/config-model/src/test/examples/simple.sd @@ -47,7 +47,6 @@ search simple { rank-type: tags stemming: none normalizing: none - header } field popularity type int { @@ -81,7 +80,6 @@ search simple { field categories type string { indexing: input categories_src | lowercase | normalize | index - body } field categoriesagain type string { diff --git a/config-model/src/test/examples/structoutsideofdocument.sd b/config-model/src/test/examples/structoutsideofdocument.sd index 4c52a248bf5..5c062ef19a6 100644 --- a/config-model/src/test/examples/structoutsideofdocument.sd +++ b/config-model/src/test/examples/structoutsideofdocument.sd @@ -11,6 +11,5 @@ search structoutsideofdocument { field nallestruct type array<nalle> { indexing: summary - body } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionsParsingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionsParsingTestCase.java index fda15528eda..fd4bb393c49 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionsParsingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionsParsingTestCase.java @@ -80,4 +80,10 @@ public class SearchDefinitionsParsingTestCase extends SearchDefinitionTestCase { } } + // TODO: Remove in Vespa 8 + @Test + public void requireThatParserHandlesHeadAndBody() throws IOException, ParseException { + assertNotNull(SearchBuilder.buildFromFile("src/test/examples/header_body.sd")); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index ac85a958ed5..89169c44079 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -252,7 +252,7 @@ public class ContainerClusterTest { } private static void addContainer(DeployLogger deployLogger, ApplicationContainerCluster cluster, String name, String hostName) { - ApplicationContainer container = new ApplicationContainer(cluster, name, 0, cluster.isHostedVespa(), cluster.getTlsSecrets()); + ApplicationContainer container = new ApplicationContainer(cluster, name, 0, cluster.isHostedVespa(), cluster.getTlsSecrets(), cluster.getTlsClientAuthority()); container.setHostResource(new HostResource(new Host(null, hostName))); container.initService(deployLogger); cluster.addContainer(container); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 8b92e1091ca..e4fcf6305ed 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -25,6 +25,7 @@ import com.yahoo.container.servlet.ServletConfigConfig; import com.yahoo.container.usability.BindingsOverviewHandler; import com.yahoo.jdisc.http.ServletPathsConfig; import com.yahoo.net.HostName; +import com.yahoo.path.Path; import com.yahoo.prelude.cluster.QrMonitorConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.AbstractService; @@ -35,13 +36,18 @@ import com.yahoo.vespa.model.container.SecretStore; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.w3c.dom.Element; import org.xml.sax.SAXException; +import java.io.File; import java.io.IOException; +import java.io.StringReader; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.logging.Level; import java.util.stream.Collectors; @@ -72,6 +78,8 @@ import static org.junit.Assert.fail; * @author gjoranv */ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { + @Rule + public TemporaryFolder applicationFolder = new TemporaryFolder(); @Test public void deprecated_jdisc_tag_is_allowed() { @@ -645,6 +653,41 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test + public void client_ca_carts_fail_with_missing_clients_pem() { + Element clusterElem = DomBuilderTest.parse( + "<container version='1.0'>", + " <client-authorize />", + "</container>"); + try { + createModel(root, clusterElem); + } catch (RuntimeException e) { + assertEquals(e.getMessage(), "client-authorize set, but security/clients.pem is missing"); + return; + } + fail(); + } + + @Test + public void client_ca_carts_succeeds_with_client_authorize_and_clients_pem() { + var applicationPackage = new MockApplicationPackage.Builder() + .withRoot(applicationFolder.getRoot()) + .build(); + + applicationPackage.getFile(Path.fromString("security")).createDirectory(); + applicationPackage.getFile(Path.fromString("security/clients.pem")).writeFile(new StringReader("I am a very nice certificate")); + + var deployState = DeployState.createTestState(applicationPackage); + + Element clusterElem = DomBuilderTest.parse( + "<container version='1.0'>", + " <client-authorize />", + "</container>"); + + createModel(root, deployState, null, clusterElem); + assertEquals(Optional.of("I am a very nice certificate"), getContainerCluster("container").getTlsClientAuthority()); + } + + @Test public void environment_vars_are_honoured() { Element clusterElem = DomBuilderTest.parse( "<container version='1.0'>", diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index c758629e99d..36dc53ac87a 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -50,6 +50,7 @@ "public boolean equals(java.lang.Object)", "public java.lang.String serializedForm()", "public java.lang.String toShortString()", + "public java.lang.String toFullString()", "public java.lang.String toString()", "public com.yahoo.config.provision.TenantName tenant()", "public com.yahoo.config.provision.ApplicationName application()", diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java index dbc57dd5abd..f175af3e2f1 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java @@ -68,11 +68,17 @@ public final class ApplicationId implements Comparable<ApplicationId> { return "tenant '" + tenant + "', application '" + application + "', instance '" + instance + "'"; } + /** Returns "dotted" string (tenant.application.instance) with instance name omitted if it is "default" */ public String toShortString() { return tenant().value() + "." + application().value() + ( instance().isDefault() ? "" : "." + instance().value() ); } + /** Returns "dotted" string (tenant.application.instance) with instance name always included */ + public String toFullString() { + return tenant().value() + "." + application().value() + "." + instance().value(); + } + private String toSerializedForm() { return tenant + ":" + application + ":" + instance; } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java index fd76dc10bdb..08f9b81fec7 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java @@ -17,4 +17,7 @@ public interface ZoneApi { default RegionName getRegionName() { return getId().region(); } CloudName getCloudName(); + + /** Returns the region name within the cloud, e.g. 'us-east-1' in AWS */ + String getCloudNativeRegionName(); } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java index 93c01ccffcd..b331fa89d32 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java @@ -59,6 +59,19 @@ public class ApplicationIdTest { } @Test + public void require_string_formats_are_correct() { + ApplicationId id1 = applicationId("foo"); + ApplicationId id2 = idFrom("bar", "baz", "default"); + ApplicationId id3 = idFrom("tenant", "baz", "bim"); + assertThat(id1.toShortString(), is("default.foo")); + assertThat(id1.toFullString(), is("default.foo.default")); + assertThat(id2.toShortString(), is("bar.baz")); + assertThat(id2.toFullString(), is("bar.baz.default")); + assertThat(id3.toShortString(), is("tenant.baz.bim")); + assertThat(id3.toFullString(), is("tenant.baz.bim")); + } + + @Test public void require_that_idstring_can_be_parsed() { ApplicationId id = ApplicationId.fromSerializedForm("ten:foo:bim"); assertThat(id.tenant().value(), is("ten")); diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index af1ced533ad..e7e626f3d22 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -63,5 +63,6 @@ sleepTimeWhenRedeployingFails long default=30 # Features (to be overridden in configserver-config.xml if needed) buildMinimalSetOfConfigModels bool default=true throwIfBootstrappingTenantRepoFails bool default=true +throwIfActiveSessionCannotBeLoaded bool default=true canReturnEmptySentinelConfig bool default=false serverNodeType enum {config, controller} default=config diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index 484124991d9..a99a2a2917d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -202,6 +202,9 @@ public class ZooKeeperClient { writeDir(app.getFile(ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR), getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR), true); + writeDir(app.getFile(ApplicationPackage.SECURITY_DIR), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.SECURITY_DIR), + true); } private void writeDir(ApplicationFile file, Path zooKeeperAppPath, boolean recurse) throws IOException { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java index 5d930472111..936b9bdefda 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.Executor; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -43,6 +44,7 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> { private static final Logger log = Logger.getLogger(RemoteSessionRepo.class.getName()); + private final GlobalComponentRegistry componentRegistry; private final Curator curator; private final Path sessionsPath; private final RemoteSessionFactory remoteSessionFactory; @@ -54,22 +56,23 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> { private final TenantApplications applicationRepo; private final Executor zkWatcherExecutor; - public RemoteSessionRepo(GlobalComponentRegistry registry, + public RemoteSessionRepo(GlobalComponentRegistry componentRegistry, RemoteSessionFactory remoteSessionFactory, ReloadHandler reloadHandler, TenantName tenantName, TenantApplications applicationRepo) { - this.curator = registry.getCurator(); + this.componentRegistry = componentRegistry; + this.curator = componentRegistry.getCurator(); this.sessionsPath = TenantRepository.getSessionsPath(tenantName); this.applicationRepo = applicationRepo; this.remoteSessionFactory = remoteSessionFactory; this.reloadHandler = reloadHandler; this.tenantName = tenantName; - this.metrics = registry.getMetrics().getOrCreateMetricUpdater(Metrics.createDimensions(tenantName)); - StripedExecutor<TenantName> zkWatcherExecutor = registry.getZkWatcherExecutor(); + this.metrics = componentRegistry.getMetrics().getOrCreateMetricUpdater(Metrics.createDimensions(tenantName)); + StripedExecutor<TenantName> zkWatcherExecutor = componentRegistry.getZkWatcherExecutor(); this.zkWatcherExecutor = command -> zkWatcherExecutor.execute(tenantName, command); initializeSessions(); - this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, registry.getZkCacheExecutor()); + this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, componentRegistry.getZkCacheExecutor()); this.directoryCache.addListener(this::childEvent); this.directoryCache.start(); } @@ -136,14 +139,19 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> { */ private void sessionAdded(long sessionId) { log.log(LogLevel.DEBUG, () -> "Adding session to RemoteSessionRepo: " + sessionId); - RemoteSession session = remoteSessionFactory.createSession(sessionId); - Path sessionPath = sessionsPath.append(String.valueOf(sessionId)); - Curator.FileCache fileCache = curator.createFileCache(sessionPath.append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH).getAbsolute(), false); - fileCache.addListener(this::nodeChanged); - loadSessionIfActive(session); - sessionStateWatchers.put(sessionId, new RemoteSessionStateWatcher(fileCache, reloadHandler, session, metrics, zkWatcherExecutor)); - addSession(session); - metrics.incAddedSessions(); + try { + RemoteSession session = remoteSessionFactory.createSession(sessionId); + Path sessionPath = sessionsPath.append(String.valueOf(sessionId)); + Curator.FileCache fileCache = curator.createFileCache(sessionPath.append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH).getAbsolute(), false); + fileCache.addListener(this::nodeChanged); + loadSessionIfActive(session); + sessionStateWatchers.put(sessionId, new RemoteSessionStateWatcher(fileCache, reloadHandler, session, metrics, zkWatcherExecutor)); + addSession(session); + metrics.incAddedSessions(); + } catch (Exception e) { + if (componentRegistry.getConfigserverConfig().throwIfActiveSessionCannotBeLoaded()) throw e; + log.log(Level.WARNING, "Failed loading session " + sessionId + ": No config for this session can be served", e); + } } private void sessionRemoved(long sessionId) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java index 0d3f2e7d0a1..33215f0a9f2 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java @@ -240,7 +240,7 @@ public class RpcFillInvoker extends FillInvoker { int skippedHits = 0; for (int i = 0; i < hits.size(); i++) { Inspector summary = summaries.entry(i).field("docsum"); - if (summary.fieldCount() != 0) { + if (summary.valid()) { hits.get(i).setField(Hit.SDDOCNAME_FIELD, documentDb.getName()); hits.get(i).addSummary(documentDb.getDocsumDefinitionSet().getDocsum(summaryClass), summary); hits.get(i).setFilled(summaryClass); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java index cd4ba191a7d..341b9b2bce3 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java @@ -201,7 +201,7 @@ public class RpcProtobufFillInvoker extends FillInvoker { int skippedHits = 0; for (int i = 0; i < hits.size(); i++) { Inspector summary = summaries.entry(i).field("docsum"); - if (summary.fieldCount() != 0) { + if (summary.valid()) { hits.get(i).setField(Hit.SDDOCNAME_FIELD, documentDb.getName()); hits.get(i).addSummary(documentDb.getDocsumDefinitionSet().getDocsum(summaryClass), summary); hits.get(i).setFilled(summaryClass); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/FillTestCase.java b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/FillTestCase.java index 6d1f19eeaf2..cc32bfe1572 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/FillTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/FillTestCase.java @@ -105,6 +105,46 @@ public class FillTestCase { assertEquals(3L, result.hits().get("hit:3").getField("field2")); assertNull(result.hits().get("hit:4").getField("field2")); + assertNull(result.hits().getError()); + } + + @Test + public void testMissingHits() { + Map<Integer, Client.NodeConnection> nodes = new HashMap<>(); + nodes.put(0, client.createConnection("host0", 123)); + nodes.put(1, client.createConnection("host1", 123)); + nodes.put(2, client.createConnection("host2", 123)); + RpcResourcePool rpcResourcePool = new RpcResourcePool(nodes); + RpcInvokerFactory factory = new RpcInvokerFactory(rpcResourcePool, null, true); + + Query query = new Query(); + Result result = new Result(query); + result.hits().add(createHit(0, 0)); + result.hits().add(createHit(2, 1)); + result.hits().add(createHit(1, 2)); + result.hits().add(createHit(2, 3)); + result.hits().add(createHit(0, 4)); + + client.setDocsumReponse("host0", 0, "summaryClass1", map("field1", "s.0.0", "field2", 0)); + client.setDocsumReponse("host2", 1, "summaryClass1", map("field1", "s.2.1", "field2", 1)); + client.setDocsumReponse("host1", 2, "summaryClass1", null); + client.setDocsumReponse("host2", 3, "summaryClass1", map("field1", "s.2.3", "field2", 3)); + client.setDocsumReponse("host0", 4, "summaryClass1", null); + + factory.createFillInvoker(db()).fill(result, "summaryClass1"); + + assertEquals("s.0.0", result.hits().get("hit:0").getField("field1").toString()); + assertEquals("s.2.1", result.hits().get("hit:1").getField("field1").toString()); + assertNull(result.hits().get("hit:2").getField("field1")); + assertEquals("s.2.3", result.hits().get("hit:3").getField("field1").toString()); + assertNull(result.hits().get("hit:4").getField("field1")); + + assertEquals(0L, result.hits().get("hit:0").getField("field2")); + assertEquals(1L, result.hits().get("hit:1").getField("field2")); + assertNull(result.hits().get("hit:2").getField("field2")); + assertEquals(3L, result.hits().get("hit:3").getField("field2")); + assertNull(result.hits().get("hit:4").getField("field2")); + assertEquals("Missing hit summary data for summary summaryClass1 for 2 hits", result.hits().getError().getDetailedMessage()); } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MockClient.java b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MockClient.java index b6b7a1f5819..2fc8c0fd620 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MockClient.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MockClient.java @@ -71,6 +71,8 @@ public class MockClient implements Client { Cursor root = responseSlime.setObject(); Cursor docsums = root.setArray("docsums"); for (Map<String, Object> docsumFields : docsumsToReturn) { + if (docsumFields == null) continue; + Cursor docsumItem = docsums.addObject(); Cursor docsum = docsumItem.setObject("docsum"); for (Map.Entry<String, Object> field : docsumFields.entrySet()) { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/AwsEventFetcher.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/AwsEventFetcher.java index d17f046c5ca..05dea1f8567 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/AwsEventFetcher.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/AwsEventFetcher.java @@ -1,11 +1,10 @@ package com.yahoo.vespa.hosted.controller.api.integration.aws; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Issue; import java.util.List; public interface AwsEventFetcher { - List<CloudEvent> getEvents(ZoneId zoneId); + List<CloudEvent> getEvents(String awsRegionName); Issue createIssue(CloudEvent event); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/AwsLimitsFetcher.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/AwsLimitsFetcher.java new file mode 100644 index 00000000000..4e76f67e7cf --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/AwsLimitsFetcher.java @@ -0,0 +1,14 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.aws; + +/** + * @author freva + */ +public interface AwsLimitsFetcher { + + /** Returns the AWS EC2 instance limits in the given AWS region */ + Ec2InstanceCounts getEc2InstanceLimits(String awsRegion); + + /** Returns the current usage of AWS EC2 instances in the given AWS region */ + Ec2InstanceCounts getEc2InstanceUsage(String awsRegion); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/Ec2InstanceCounts.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/Ec2InstanceCounts.java new file mode 100644 index 00000000000..044789f14e4 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/Ec2InstanceCounts.java @@ -0,0 +1,49 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.aws; + +import java.util.Map; +import java.util.Objects; + +/** + * @author freva + */ +public class Ec2InstanceCounts { + private final int totalCount; + private final Map<String, Integer> instanceCounts; + + public Ec2InstanceCounts(int totalCount, Map<String, Integer> instanceCounts) { + this.totalCount = totalCount; + this.instanceCounts = Map.copyOf(instanceCounts); + } + + public int getTotalCount() { + return totalCount; + } + + /** Returns map of counts by instance type, e.g. 'r5.2xlarge' */ + public Map<String, Integer> getInstanceCounts() { + return instanceCounts; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Ec2InstanceCounts that = (Ec2InstanceCounts) o; + return totalCount == that.totalCount && + instanceCounts.equals(that.instanceCounts); + } + + @Override + public int hashCode() { + return Objects.hash(totalCount, instanceCounts); + } + + @Override + public String toString() { + return "Ec2InstanceLimits{" + + "totalLimit=" + totalCount + + ", instanceCounts=" + instanceCounts + + '}'; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockAwsEventFetcher.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockAwsEventFetcher.java index 73b1942de44..b0b06fc6c83 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockAwsEventFetcher.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockAwsEventFetcher.java @@ -1,6 +1,5 @@ package com.yahoo.vespa.hosted.controller.api.integration.aws; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Issue; import java.util.List; @@ -8,7 +7,7 @@ import java.util.Optional; public class MockAwsEventFetcher implements AwsEventFetcher { @Override - public List<CloudEvent> getEvents(ZoneId zoneId) { + public List<CloudEvent> getEvents(String awsRegionName) { return List.of(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 7ee834844af..ee559cef6d9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -588,7 +588,9 @@ public class InternalStepRunner implements StepRunner { ApplicationVersion version = controller.jobController().run(id).get().versions().targetApplication(); DeploymentSpec spec = controller.applications().require(id.application()).deploymentSpec(); - byte[] servicesXml = servicesXml(controller.zoneRegistry().accessControlDomain(), testerFlavorFor(id, spec)); + byte[] servicesXml = servicesXml(controller.zoneRegistry().accessControlDomain(), + spec.athenzDomain().isPresent(), + testerFlavorFor(id, spec)); byte[] testPackage = controller.applications().applicationStore().get(id.tester(), version); ZoneId zone = id.type().zone(controller.system()); @@ -620,7 +622,7 @@ public class InternalStepRunner implements StepRunner { } /** Returns the generated services.xml content for the tester application. */ - static byte[] servicesXml(AthenzDomain domain, Optional<String> testerFlavor) { + static byte[] servicesXml(AthenzDomain domain, boolean useAthenzCredentials, Optional<String> testerFlavor) { String flavor = testerFlavor.orElse("d-1-4-50"); int memoryGb = Integer.parseInt(flavor.split("-")[2]); // Memory available in tester container. int jdiscMemoryPercentage = (int) Math.ceil(200.0 / memoryGb); // 2Gb memory for tester application (excessive?). @@ -635,6 +637,7 @@ public class InternalStepRunner implements StepRunner { " <config name=\"com.yahoo.vespa.hosted.testrunner.test-runner\">\n" + " <artifactsPath>artifacts</artifactsPath>\n" + " <surefireMemoryMb>" + testMemoryMb + "</surefireMemoryMb>\n" + + " <useAthenzCredentials>" + useAthenzCredentials + "</useAthenzCredentials>\n" + " </config>\n" + " </component>\n" + "\n" + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/AwsEventReporterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/AwsEventReporterMaintainer.java index 14a39109c61..12bee2a7954 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/AwsEventReporterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/AwsEventReporterMaintainer.java @@ -1,7 +1,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.zone.ZoneList; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.aws.AwsEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.CloudEvent; @@ -10,8 +10,10 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueHandl import java.time.Duration; import java.util.List; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; /** * @author mgimle @@ -22,27 +24,26 @@ public class AwsEventReporterMaintainer extends Maintainer { private final IssueHandler issueHandler; private final AwsEventFetcher eventFetcher; - private final ZoneList cloudZones; + private final Set<String> awsRegions; AwsEventReporterMaintainer(Controller controller, Duration interval, JobControl jobControl, IssueHandler issueHandler, AwsEventFetcher eventFetcher) { super(controller, interval, jobControl); - this.cloudZones = awsZones(controller); this.issueHandler = issueHandler; this.eventFetcher = eventFetcher; - } - - private ZoneList awsZones(Controller controller) { - return controller.zoneRegistry().zones() + this.awsRegions = controller.zoneRegistry().zones() .ofCloud(CloudName.from("aws")) - .reachable(); + .reachable() + .zones().stream() + .map(ZoneApi::getCloudNativeRegionName) + .collect(Collectors.toSet()); } @Override protected void maintain() { log.log(Level.INFO, "Fetching events for cloud hosts."); - for (var cloudZoneId : cloudZones.ids()) { - List<CloudEvent> events = eventFetcher.getEvents(cloudZoneId); + for (var awsRegion : awsRegions) { + List<CloudEvent> events = eventFetcher.getEvents(awsRegion); for (var event : events) { Issue issue = eventFetcher.createIssue(event); if (!issueHandler.issueExists(issue)) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index f8157680cfd..f5366ae3862 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -399,7 +399,9 @@ public class InternalStepRunnerTest { @Test public void generates_correct_services_xml_test() { - assertFile("test_runner_services.xml-cd", new String(InternalStepRunner.servicesXml(AthenzDomain.from("vespa.vespa.cd"), Optional.of("d-2-12-75")))); + assertFile("test_runner_services.xml-cd", new String(InternalStepRunner.servicesXml(AthenzDomain.from("vespa.vespa.cd"), + true, + Optional.of("d-2-12-75")))); } private void assertFile(String resourceName, String actualContent) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java index 4705982e1f2..269bdcc5dca 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java @@ -1,15 +1,12 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; -import com.yahoo.cloud.config.SentinelConfig; -import com.yahoo.config.model.graph.ModelGraphBuilder; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.messagebus.MessagebusConfig; /** * @author hakonhall @@ -18,13 +15,15 @@ public class ZoneApiMock implements ZoneApi { private final SystemName systemName; private final ZoneId id; private final CloudName cloudName; + private final String cloudNativeRegionName; public static Builder newBuilder() { return new Builder(); } - private ZoneApiMock(SystemName systemName, ZoneId id, CloudName cloudName) { + private ZoneApiMock(SystemName systemName, ZoneId id, CloudName cloudName, String cloudNativeRegionName) { this.systemName = systemName; this.id = id; this.cloudName = cloudName; + this.cloudNativeRegionName = cloudNativeRegionName; } public static ZoneApiMock fromId(String id) { @@ -44,10 +43,14 @@ public class ZoneApiMock implements ZoneApi { @Override public CloudName getCloudName() { return cloudName; } + @Override + public String getCloudNativeRegionName() { return cloudNativeRegionName; } + public static class Builder { private SystemName systemName = SystemName.defaultSystem(); private ZoneId id = ZoneId.defaultId(); private CloudName cloudName = CloudName.defaultName(); + private String cloudNativeRegionName = id.region().value(); public Builder with(ZoneId id) { this.id = id; @@ -63,8 +66,13 @@ public class ZoneApiMock implements ZoneApi { public Builder withCloud(String cloud) { return with(CloudName.from(cloud)); } + public Builder withCloudNativeRegionName(String cloudRegionName) { + this.cloudNativeRegionName = cloudRegionName; + return this; + } + public ZoneApiMock build() { - return new ZoneApiMock(systemName, id, cloudName); + return new ZoneApiMock(systemName, id, cloudName, cloudNativeRegionName); } } } diff --git a/controller-server/src/test/resources/test_runner_services.xml-cd b/controller-server/src/test/resources/test_runner_services.xml-cd index bd49f3be55a..f53262fe55f 100644 --- a/controller-server/src/test/resources/test_runner_services.xml-cd +++ b/controller-server/src/test/resources/test_runner_services.xml-cd @@ -6,6 +6,7 @@ <config name="com.yahoo.vespa.hosted.testrunner.test-runner"> <artifactsPath>artifacts</artifactsPath> <surefireMemoryMb>5120</surefireMemoryMb> + <useAthenzCredentials>true</useAthenzCredentials> </config> </component> diff --git a/jdisc_http_service/abi-spec.json b/jdisc_http_service/abi-spec.json index f915dc1e8c1..6e04ba741e6 100644 --- a/jdisc_http_service/abi-spec.json +++ b/jdisc_http_service/abi-spec.json @@ -39,6 +39,7 @@ "public com.yahoo.jdisc.http.ConnectorConfig$Builder tcpNoDelay(boolean)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder throttling(com.yahoo.jdisc.http.ConnectorConfig$Throttling$Builder)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder ssl(com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder)", + "public com.yahoo.jdisc.http.ConnectorConfig$Builder tlsClientAuthEnforcer(com.yahoo.jdisc.http.ConnectorConfig$TlsClientAuthEnforcer$Builder)", "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", "public final java.lang.String getDefMd5()", "public final java.lang.String getDefName()", @@ -47,7 +48,8 @@ ], "fields": [ "public com.yahoo.jdisc.http.ConnectorConfig$Throttling$Builder throttling", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder ssl" + "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder ssl", + "public com.yahoo.jdisc.http.ConnectorConfig$TlsClientAuthEnforcer$Builder tlsClientAuthEnforcer" ] }, "com.yahoo.jdisc.http.ConnectorConfig$Producer": { @@ -82,6 +84,7 @@ "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder certificateFile(java.lang.String)", "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder certificate(java.lang.String)", "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder caCertificateFile(java.lang.String)", + "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder caCertificate(java.lang.String)", "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder clientAuth(com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum)", "public com.yahoo.jdisc.http.ConnectorConfig$Ssl build()" ], @@ -137,6 +140,7 @@ "public java.lang.String certificateFile()", "public java.lang.String certificate()", "public java.lang.String caCertificateFile()", + "public java.lang.String caCertificate()", "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum clientAuth()" ], "fields": [] @@ -178,6 +182,41 @@ ], "fields": [] }, + "com.yahoo.jdisc.http.ConnectorConfig$TlsClientAuthEnforcer$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigBuilder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$TlsClientAuthEnforcer)", + "public com.yahoo.jdisc.http.ConnectorConfig$TlsClientAuthEnforcer$Builder enable(boolean)", + "public com.yahoo.jdisc.http.ConnectorConfig$TlsClientAuthEnforcer$Builder pathWhitelist(java.lang.String)", + "public com.yahoo.jdisc.http.ConnectorConfig$TlsClientAuthEnforcer$Builder pathWhitelist(java.util.Collection)", + "public com.yahoo.jdisc.http.ConnectorConfig$TlsClientAuthEnforcer build()" + ], + "fields": [ + "public java.util.List pathWhitelist" + ] + }, + "com.yahoo.jdisc.http.ConnectorConfig$TlsClientAuthEnforcer": { + "superClass": "com.yahoo.config.InnerNode", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$TlsClientAuthEnforcer$Builder)", + "public boolean enable()", + "public java.util.List pathWhitelist()", + "public java.lang.String pathWhitelist(int)" + ], + "fields": [] + }, "com.yahoo.jdisc.http.ConnectorConfig": { "superClass": "com.yahoo.config.ConfigInstance", "interfaces": [], @@ -204,7 +243,8 @@ "public boolean tcpKeepAliveEnabled()", "public boolean tcpNoDelay()", "public com.yahoo.jdisc.http.ConnectorConfig$Throttling throttling()", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl ssl()" + "public com.yahoo.jdisc.http.ConnectorConfig$Ssl ssl()", + "public com.yahoo.jdisc.http.ConnectorConfig$TlsClientAuthEnforcer tlsClientAuthEnforcer()" ], "fields": [ "public static final java.lang.String CONFIG_DEF_MD5", diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java index 30a1b1d885c..2b9cb426dda 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java @@ -9,6 +9,7 @@ import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.logging.AccessLog; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.application.OsgiFramework; +import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.jdisc.http.ServerConfig; import com.yahoo.jdisc.http.ServletPathsConfig; import com.yahoo.jdisc.http.server.FilterBindings; @@ -145,10 +146,13 @@ public class JettyHttpServer extends AbstractServerProvider { setupJmx(server, serverConfig); ((QueuedThreadPool)server.getThreadPool()).setMaxThreads(serverConfig.maxWorkerThreads()); + List<ConnectorConfig> connectorConfigs = new ArrayList<>(); for (ConnectorFactory connectorFactory : connectorFactories.allComponents()) { - ServerSocketChannel preBoundChannel = getChannelFromServiceLayer(connectorFactory.getConnectorConfig().listenPort(), osgiFramework.bundleContext()); + ConnectorConfig connectorConfig = connectorFactory.getConnectorConfig(); + connectorConfigs.add(connectorConfig); + ServerSocketChannel preBoundChannel = getChannelFromServiceLayer(connectorConfig.listenPort(), osgiFramework.bundleContext()); server.addConnector(connectorFactory.createConnector(metric, server, preBoundChannel)); - listenedPorts.add(connectorFactory.getConnectorConfig().listenPort()); + listenedPorts.add(connectorConfig.listenPort()); } janitor = newJanitor(threadFactory); @@ -168,6 +172,7 @@ public class JettyHttpServer extends AbstractServerProvider { getHandlerCollection( serverConfig, servletPathsConfig, + connectorConfigs, jdiscServlet, servletHolders, jDiscFilterInvokerFilter)); @@ -217,6 +222,7 @@ public class JettyHttpServer extends AbstractServerProvider { private HandlerCollection getHandlerCollection( ServerConfig serverConfig, ServletPathsConfig servletPathsConfig, + List<ConnectorConfig> connectorConfigs, ServletHolder jdiscServlet, ComponentRegistry<ServletHolder> servletHolders, FilterHolder jDiscFilterInvokerFilter) { @@ -231,8 +237,11 @@ public class JettyHttpServer extends AbstractServerProvider { servletContextHandler.addServlet(jdiscServlet, "/*"); + var authEnforcer = new TlsClientAuthenticationEnforcer(connectorConfigs); + authEnforcer.setHandler(servletContextHandler); + GzipHandler gzipHandler = newGzipHandler(serverConfig); - gzipHandler.setHandler(servletContextHandler); + gzipHandler.setHandler(authEnforcer); HttpResponseStatisticsCollector statisticsCollector = new HttpResponseStatisticsCollector(); statisticsCollector.setHandler(gzipHandler); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/TlsClientAuthenticationEnforcer.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/TlsClientAuthenticationEnforcer.java new file mode 100644 index 00000000000..546741b3322 --- /dev/null +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/TlsClientAuthenticationEnforcer.java @@ -0,0 +1,78 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.server.jetty; + +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.jdisc.http.servlet.ServletRequest; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.handler.HandlerWrapper; + +import javax.servlet.DispatcherType; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * A Jetty handler that enforces TLS client authentication with configurable white list. + * + * @author bjorncs + */ +class TlsClientAuthenticationEnforcer extends HandlerWrapper { + + private final Map<Integer, List<String>> portToWhitelistedPathsMapping; + + TlsClientAuthenticationEnforcer(List<ConnectorConfig> connectorConfigs) { + portToWhitelistedPathsMapping = createWhitelistMapping(connectorConfigs); + } + + @Override + public void handle(String target, Request request, HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws IOException, ServletException { + if (isHttpsRequest(request) + && !isRequestToWhitelistedBinding(servletRequest) + && !isClientAuthenticated(servletRequest)) { + servletResponse.sendError(Response.Status.UNAUTHORIZED, "Client did not present a x509 certificate."); + } else { + _handler.handle(target, request, servletRequest, servletResponse); + } + } + + private static Map<Integer, List<String>> createWhitelistMapping(List<ConnectorConfig> connectorConfigs) { + var mapping = new HashMap<Integer, List<String>>(); + for (ConnectorConfig connectorConfig : connectorConfigs) { + var enforcerConfig = connectorConfig.tlsClientAuthEnforcer(); + if (enforcerConfig.enable()) { + mapping.put(connectorConfig.listenPort(), enforcerConfig.pathWhitelist()); + } + } + return mapping; + } + + private boolean isHttpsRequest(Request request) { + return request.getDispatcherType() == DispatcherType.REQUEST && request.getScheme().equalsIgnoreCase("https"); + } + + private boolean isRequestToWhitelistedBinding(HttpServletRequest servletRequest) { + int localPort = servletRequest.getLocalPort(); + List<String> whiteListedPaths = getWhitelistedPathsForPort(localPort); + if (whiteListedPaths == null) { + return true; // enforcer not enabled + } + // Note: Same path definition as HttpRequestFactory.getUri() + return whiteListedPaths.contains(servletRequest.getRequestURI()); + } + + private List<String> getWhitelistedPathsForPort(int localPort) { + if (portToWhitelistedPathsMapping.containsKey(0) && portToWhitelistedPathsMapping.size() == 1) { + return portToWhitelistedPathsMapping.get(0); // for unit tests which uses 0 for listen port + } + return portToWhitelistedPathsMapping.get(localPort); + } + + private boolean isClientAuthenticated(HttpServletRequest servletRequest) { + return servletRequest.getAttribute(ServletRequest.SERVLET_REQUEST_X509CERT) != null; + } +} diff --git a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def index c6c6fad345b..f02a0d7b4a3 100644 --- a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def +++ b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def @@ -69,7 +69,22 @@ ssl.certificateFile string default="" ssl.certificate string default="" # with trusted CA certificates in PEM format. Used to verify clients +# - this is the name of a file on the local container file system +# - only one of caCertificateFile and caCertificate ssl.caCertificateFile string default="" +# with trusted CA certificates in PEM format. Used to verify clients +# - this is the actual certificates instead of a pointer to the file +# - only one of caCertificateFile and caCertificate +ssl.caCertificate string default="" + # Client authentication mode. See SSLEngine.getNeedClientAuth()/getWantClientAuth() for details. ssl.clientAuth enum { DISABLED, WANT_AUTH, NEED_AUTH } default=DISABLED + +# Enforce TLS client authentication for https requests at the http layer. +# Intended to be used with connectors with optional client authentication enabled. +# 401 status code is returned for requests from non-authenticated clients. +tlsClientAuthEnforcer.enable bool default=false + +# Paths where client authentication should not be enforced. To be used in combination with WANT_AUTH. Typically used for health checks. +tlsClientAuthEnforcer.pathWhitelist[] string diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java index ec9c90ffa50..31ecf3ca2fc 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java @@ -23,6 +23,7 @@ import com.yahoo.jdisc.http.HttpResponse; import com.yahoo.jdisc.http.ServerConfig; import com.yahoo.jdisc.service.BindingSetNotFoundException; import com.yahoo.security.KeyUtils; +import com.yahoo.security.SslContextBuilder; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.X509CertificateUtils; import org.apache.http.entity.ContentType; @@ -32,7 +33,9 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import javax.net.ssl.SSLContext; import javax.security.auth.x500.X500Principal; +import java.io.IOException; import java.math.BigInteger; import java.net.BindException; import java.net.URI; @@ -58,6 +61,7 @@ import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; import static com.yahoo.jdisc.Response.Status.NOT_FOUND; import static com.yahoo.jdisc.Response.Status.OK; import static com.yahoo.jdisc.Response.Status.REQUEST_URI_TOO_LONG; +import static com.yahoo.jdisc.Response.Status.UNAUTHORIZED; import static com.yahoo.jdisc.Response.Status.UNSUPPORTED_MEDIA_TYPE; import static com.yahoo.jdisc.http.HttpHeaders.Names.CONNECTION; import static com.yahoo.jdisc.http.HttpHeaders.Names.CONTENT_TYPE; @@ -470,16 +474,9 @@ public class HttpServerTest { @Test public void requireThatServerCanRespondToSslRequest() throws Exception { - KeyPair keyPair = KeyUtils.generateKeypair(RSA, 2048); Path privateKeyFile = tmpFolder.newFile().toPath(); - Files.writeString(privateKeyFile, KeyUtils.toPem(keyPair.getPrivate())); - - X509Certificate certificate = X509CertificateBuilder - .fromKeypair( - keyPair, new X500Principal("CN=localhost"), Instant.EPOCH, Instant.EPOCH.plus(100_000, ChronoUnit.DAYS), SHA256_WITH_RSA, BigInteger.ONE) - .build(); Path certificateFile = tmpFolder.newFile().toPath(); - Files.writeString(certificateFile, X509CertificateUtils.toPem(certificate)); + generatePrivateKeyAndCertificate(privateKeyFile, certificateFile); final TestDriver driver = TestDrivers.newInstanceWithSsl(new EchoRequestHandler(), certificateFile, privateKeyFile); driver.client().get("/status.html") @@ -488,6 +485,42 @@ public class HttpServerTest { } @Test + public void requireThatTlsClientAuthenticationEnforcerRejectsRequestsForNonWhitelistedPaths() throws IOException { + Path privateKeyFile = tmpFolder.newFile().toPath(); + Path certificateFile = tmpFolder.newFile().toPath(); + generatePrivateKeyAndCertificate(privateKeyFile, certificateFile); + TestDriver driver = TestDrivers.newInstanceWithSsl(new EchoRequestHandler(), certificateFile, privateKeyFile); + + SSLContext trustStoreOnlyCtx = new SslContextBuilder() + .withTrustStore(certificateFile) + .build(); + + new SimpleHttpClient(trustStoreOnlyCtx, driver.server().getListenPort(), false) + .get("/dummy.html") + .expectStatusCode(is(UNAUTHORIZED)); + + assertThat(driver.close(), is(true)); + } + + @Test + public void requireThatTlsClientAuthenticationEnforcerAllowsRequestForWhitelistedPaths() throws IOException { + Path privateKeyFile = tmpFolder.newFile().toPath(); + Path certificateFile = tmpFolder.newFile().toPath(); + generatePrivateKeyAndCertificate(privateKeyFile, certificateFile); + TestDriver driver = TestDrivers.newInstanceWithSsl(new EchoRequestHandler(), certificateFile, privateKeyFile); + + SSLContext trustStoreOnlyCtx = new SslContextBuilder() + .withTrustStore(certificateFile) + .build(); + + new SimpleHttpClient(trustStoreOnlyCtx, driver.server().getListenPort(), false) + .get("/status.html") + .expectStatusCode(is(OK)); + + assertThat(driver.close(), is(true)); + } + + @Test public void requireThatConnectedAtReturnsNonZero() throws Exception { final TestDriver driver = TestDrivers.newInstance(new ConnectedAtRequestHandler()); driver.client().get("/status.html") @@ -526,6 +559,17 @@ public class HttpServerTest { assertThat(driver.close(), is(true)); } + private static void generatePrivateKeyAndCertificate(Path privateKeyFile, Path certificateFile) throws IOException { + KeyPair keyPair = KeyUtils.generateKeypair(RSA, 2048); + Files.writeString(privateKeyFile, KeyUtils.toPem(keyPair.getPrivate())); + + X509Certificate certificate = X509CertificateBuilder + .fromKeypair( + keyPair, new X500Principal("CN=localhost"), Instant.EPOCH, Instant.EPOCH.plus(100_000, ChronoUnit.DAYS), SHA256_WITH_RSA, BigInteger.ONE) + .build(); + Files.writeString(certificateFile, X509CertificateUtils.toPem(certificate)); + } + private static RequestHandler mockRequestHandler() { final RequestHandler mockRequestHandler = mock(RequestHandler.class); when(mockRequestHandler.refer()).thenReturn(References.NOOP_REFERENCE); diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDrivers.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDrivers.java index 10fe0f1328f..e0933ac485e 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDrivers.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDrivers.java @@ -55,8 +55,13 @@ public class TestDrivers { newConfigModule( new ServerConfig.Builder(), new ConnectorConfig.Builder() + .tlsClientAuthEnforcer( + new ConnectorConfig.TlsClientAuthEnforcer.Builder() + .enable(true) + .pathWhitelist("/status.html")) .ssl(new ConnectorConfig.Ssl.Builder() .enabled(true) + .clientAuth(ConnectorConfig.Ssl.ClientAuth.Enum.WANT_AUTH) .privateKeyFile(privateKeyFile.toString()) .certificateFile(certificateFile.toString()) .caCertificateFile(certificateFile.toString())), diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java index e20480e14ef..b2769ee40c4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java @@ -254,6 +254,11 @@ public class NodeAgentContextImpl implements NodeAgentContext { public CloudName getCloudName() { return CloudName.defaultName(); } + + @Override + public String getCloudNativeRegionName() { + return getId().region().value(); + } }), Optional.ofNullable(pathToContainerStorage).orElseGet(() -> Paths.get("/home/docker")), Optional.ofNullable(pathToVespaHome).orElseGet(() -> Paths.get("/opt/vespa")), diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java index fb5dccc551d..e97956b71d5 100644 --- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java @@ -16,6 +16,7 @@ import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.SortedMap; @@ -68,30 +69,7 @@ public class TestRunner { vespaHome.resolve("logs/vespa/maven.log"), vespaHome.resolve("tmp/config.json"), vespaHome.resolve("tmp/settings.xml"), - profile -> { // Anything to make this testable! >_< - String[] command = new String[]{ - "mvn", - "test", - - "--batch-mode", // Run in non-interactive (batch) mode (disables output color) - "--show-version", // Display version information WITHOUT stopping build - "--settings", // Need to override repository settings in ymaven config >_< - vespaHome.resolve("tmp/settings.xml").toString(), - - // Disable maven download progress indication - "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn", - "-Dstyle.color=always", // Enable ANSI color codes again - "-DfailIfNoTests=" + profile.failIfNoTests(), - "-Dvespa.test.config=" + vespaHome.resolve("tmp/config.json"), - "-Dvespa.test.credentials.root=" + Defaults.getDefaults().vespaHome() + "/var/vespa/sia", - String.format("-DargLine=-Xms%1$dm -Xmx%1$dm", config.surefireMemoryMb()) - }; - ProcessBuilder builder = new ProcessBuilder(command); - builder.environment().merge("MAVEN_OPTS", " -Djansi.force=true", String::concat); - builder.directory(vespaHome.resolve("tmp/test").toFile()); - builder.redirectErrorStream(true); - return builder; - }); + profile -> mavenProcessFrom(profile, config)); } TestRunner(Path artifactsPath, Path testPath, Path logFile, Path configFile, Path settingsFile, Function<TestProfile, ProcessBuilder> testBuilder) { @@ -103,6 +81,33 @@ public class TestRunner { this.testBuilder = testBuilder; } + static ProcessBuilder mavenProcessFrom(TestProfile profile, TestRunnerConfig config) { + List<String> command = new ArrayList<>(); + command.add("mvn"); + command.add("test"); + + command.add("--batch-mode"); // Run in non-interactive (batch) mode (disables output color) + command.add("--show-version"); // Display version information WITHOUT stopping build + command.add("--settings"); // Need to override repository settings in ymaven config >_< + command.add(vespaHome.resolve("tmp/settings.xml").toString()); + + // Disable maven download progress indication + command.add("-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"); + command.add("-Dstyle.color=always"); // Enable ANSI color codes again + command.add("-DfailIfNoTests=" + profile.failIfNoTests()); + command.add("-Dvespa.test.config=" + vespaHome.resolve("tmp/config.json")); + if (config.useAthenzCredentials()) + command.add("-Dvespa.test.credentials.root=" + Defaults.getDefaults().vespaHome() + "/var/vespa/sia"); + command.add(String.format("-DargLine=-Xms%1$dm -Xmx%1$dm", config.surefireMemoryMb())); + + ProcessBuilder builder = new ProcessBuilder(command); + builder.environment().merge("MAVEN_OPTS", " -Djansi.force=true", String::concat); + builder.environment().merge("PATH", ":" + vespaHome.resolve("local/maven/bin/"), String::concat); + builder.directory(vespaHome.resolve("tmp/test").toFile()); + builder.redirectErrorStream(true); + return builder; + } + public synchronized void test(TestProfile testProfile, byte[] testConfig) { if (status == Status.RUNNING) throw new IllegalArgumentException("Tests are already running; should not receive this request now."); diff --git a/vespa-testrunner-components/src/main/resources/configdefinitions/test-runner.def b/vespa-testrunner-components/src/main/resources/configdefinitions/test-runner.def index a2d0eacd9be..91a8a2057e7 100644 --- a/vespa-testrunner-components/src/main/resources/configdefinitions/test-runner.def +++ b/vespa-testrunner-components/src/main/resources/configdefinitions/test-runner.def @@ -1,4 +1,5 @@ package=com.yahoo.vespa.hosted.testrunner -artifactsPath path -surefireMemoryMb int +artifactsPath path +surefireMemoryMb int +useAthenzCredentials bool diff --git a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java index 49c95fa4b6f..e8fc589b972 100644 --- a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java +++ b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java @@ -15,7 +15,6 @@ import java.util.logging.LogRecord; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** * Unit tests relying on a UNIX shell >_< diff --git a/vespalib/src/tests/datastore/unique_store/CMakeLists.txt b/vespalib/src/tests/datastore/unique_store/CMakeLists.txt index dd200018448..d72e8c10ad5 100644 --- a/vespalib/src/tests/datastore/unique_store/CMakeLists.txt +++ b/vespalib/src/tests/datastore/unique_store/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(vespalib_unique_store_test_app TEST unique_store_test.cpp DEPENDS vespalib + gtest ) vespa_add_test(NAME vespalib_unique_store_test_app COMMAND vespalib_unique_store_test_app) diff --git a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp index 2ed7bfd52ed..a585186aa3e 100644 --- a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp +++ b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp @@ -1,13 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("unique_store_test"); #include <vespa/vespalib/datastore/unique_store.hpp> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/test/datastore/memstats.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/util/traits.h> #include <vector> +#include <vespa/log/log.h> +LOG_SETUP("unique_store_test"); + using namespace search::datastore; using vespalib::MemoryUsage; using vespalib::ArrayRef; @@ -15,8 +16,7 @@ using generation_t = vespalib::GenerationHandler::generation_t; using MemStats = search::datastore::test::MemStats; template <typename EntryT, typename RefT = EntryRefT<22> > -struct Fixture -{ +struct TestBase : public ::testing::Test { using EntryRefType = RefT; using UniqueStoreType = UniqueStore<EntryT, RefT>; using value_type = EntryT; @@ -25,7 +25,7 @@ struct Fixture UniqueStoreType store; ReferenceStore refStore; generation_t generation; - Fixture() + TestBase() : store(), refStore(), generation(1) @@ -38,7 +38,7 @@ struct Fixture UniqueStoreAddResult addResult = store.add(input); EntryRef result = addResult.ref(); auto insres = refStore.insert(std::make_pair(result, std::make_pair(input, 1u))); - EXPECT_EQUAL(insres.second, addResult.inserted()); + EXPECT_EQ(insres.second, addResult.inserted()); if (!insres.second) { ++insres.first->second.second; } @@ -56,10 +56,10 @@ struct Fixture } void assertGet(EntryRef ref, const EntryT &exp) const { EntryT act = store.get(ref); - EXPECT_EQUAL(exp, act); + EXPECT_EQ(exp, act); } void remove(EntryRef ref) { - ASSERT_EQUAL(1u, refStore.count(ref)); + ASSERT_EQ(1u, refStore.count(ref)); store.remove(ref); if (refStore[ref].second > 1) { --refStore[ref].second; @@ -74,19 +74,19 @@ struct Fixture return EntryRefType(ref).bufferId(); } void assertBufferState(EntryRef ref, const MemStats expStats) const { - EXPECT_EQUAL(expStats._used, store.bufferState(ref).size()); - EXPECT_EQUAL(expStats._hold, store.bufferState(ref).getHoldElems()); - EXPECT_EQUAL(expStats._dead, store.bufferState(ref).getDeadElems()); + EXPECT_EQ(expStats._used, store.bufferState(ref).size()); + EXPECT_EQ(expStats._hold, store.bufferState(ref).getHoldElems()); + EXPECT_EQ(expStats._dead, store.bufferState(ref).getDeadElems()); } void assertMemoryUsage(const MemStats expStats) const { MemoryUsage act = store.getMemoryUsage(); - EXPECT_EQUAL(expStats._used, act.usedBytes()); - EXPECT_EQUAL(expStats._hold, act.allocatedBytesOnHold()); - EXPECT_EQUAL(expStats._dead, act.deadBytes()); + EXPECT_EQ(expStats._used, act.usedBytes()); + EXPECT_EQ(expStats._hold, act.allocatedBytesOnHold()); + EXPECT_EQ(expStats._dead, act.deadBytes()); } void assertStoreContent() const { for (const auto &elem : refStore) { - TEST_DO(assertGet(elem.first, elem.second.first)); + assertGet(elem.first, elem.second.first); } } EntryRef getEntryRef(const EntryT &input) { @@ -115,8 +115,8 @@ struct Fixture refs.pop_back(); ReferenceStore compactedRefStore; for (size_t i = 0; i < refs.size(); ++i) { - ASSERT_EQUAL(0u, compactedRefStore.count(compactedRefs[i])); - ASSERT_EQUAL(1u, refStore.count(refs[i])); + ASSERT_EQ(0u, compactedRefStore.count(compactedRefs[i])); + ASSERT_EQ(1u, refStore.count(refs[i])); compactedRefStore.insert(std::make_pair(compactedRefs[i], refStore[refs[i]])); } refStore = compactedRefStore; @@ -126,141 +126,141 @@ struct Fixture auto getSaver() { return store.getSaver(); } }; -using NumberFixture = Fixture<uint32_t>; -using StringFixture = Fixture<std::string>; -using SmallOffsetNumberFixture = Fixture<uint32_t, EntryRefT<10>>; +using NumberTest = TestBase<uint32_t>; +using StringTest = TestBase<std::string>; +using SmallOffsetNumberTest = TestBase<uint32_t, EntryRefT<10>>; -TEST("require that we test with trivial and non-trivial types") +TEST(UniqueStoreTest, trivial_and_non_trivial_types_are_tested) { - EXPECT_TRUE(vespalib::can_skip_destruction<NumberFixture::value_type>::value); - EXPECT_FALSE(vespalib::can_skip_destruction<StringFixture::value_type>::value); + EXPECT_TRUE(vespalib::can_skip_destruction<NumberTest::value_type>::value); + EXPECT_FALSE(vespalib::can_skip_destruction<StringTest::value_type>::value); } -TEST_F("require that we can add and get values of trivial type", NumberFixture) +TEST_F(NumberTest, can_add_and_get_values_of_trivial_type) { - TEST_DO(f.assertAdd(1)); - TEST_DO(f.assertAdd(2)); - TEST_DO(f.assertAdd(3)); - TEST_DO(f.assertAdd(1)); + assertAdd(1); + assertAdd(2); + assertAdd(3); + assertAdd(1); } -TEST_F("require that we can add and get values of non-trivial type", StringFixture) +TEST_F(StringTest, can_add_and_get_values_of_non_trivial_type) { - TEST_DO(f.assertAdd("aa")); - TEST_DO(f.assertAdd("bbb")); - TEST_DO(f.assertAdd("ccc")); - TEST_DO(f.assertAdd("aa")); + assertAdd("aa"); + assertAdd("bbb"); + assertAdd("ccc"); + assertAdd("aa"); } -TEST_F("require that elements are put on hold when value is removed", NumberFixture) +TEST_F(NumberTest, elements_are_put_on_hold_when_value_is_removed) { - EntryRef ref = f.add(1); + EntryRef ref = add(1); // Note: The first buffer have the first element reserved -> we expect 2 elements used here. - TEST_DO(f.assertBufferState(ref, MemStats().used(2).hold(0).dead(1))); - f.store.remove(ref); - TEST_DO(f.assertBufferState(ref, MemStats().used(2).hold(1).dead(1))); + assertBufferState(ref, MemStats().used(2).hold(0).dead(1)); + store.remove(ref); + assertBufferState(ref, MemStats().used(2).hold(1).dead(1)); } -TEST_F("require that elements are reference counted", NumberFixture) +TEST_F(NumberTest, elements_are_reference_counted) { - EntryRef ref = f.add(1); - EntryRef ref2 = f.add(1); - EXPECT_EQUAL(ref.ref(), ref2.ref()); + EntryRef ref = add(1); + EntryRef ref2 = add(1); + EXPECT_EQ(ref.ref(), ref2.ref()); // Note: The first buffer have the first element reserved -> we expect 2 elements used here. - TEST_DO(f.assertBufferState(ref, MemStats().used(2).hold(0).dead(1))); - f.store.remove(ref); - TEST_DO(f.assertBufferState(ref, MemStats().used(2).hold(0).dead(1))); - f.store.remove(ref); - TEST_DO(f.assertBufferState(ref, MemStats().used(2).hold(1).dead(1))); + assertBufferState(ref, MemStats().used(2).hold(0).dead(1)); + store.remove(ref); + assertBufferState(ref, MemStats().used(2).hold(0).dead(1)); + store.remove(ref); + assertBufferState(ref, MemStats().used(2).hold(1).dead(1)); } -TEST_F("require that new underlying buffer is allocated when current is full", SmallOffsetNumberFixture) +TEST_F(SmallOffsetNumberTest, new_underlying_buffer_is_allocated_when_current_is_full) { - uint32_t firstBufferId = f.getBufferId(f.add(1)); - for (uint32_t i = 0; i < (F1::EntryRefType::offsetSize() - 2); ++i) { - uint32_t bufferId = f.getBufferId(f.add(i + 2)); - EXPECT_EQUAL(firstBufferId, bufferId); + uint32_t firstBufferId = getBufferId(add(1)); + for (uint32_t i = 0; i < (SmallOffsetNumberTest::EntryRefType::offsetSize() - 2); ++i) { + uint32_t bufferId = getBufferId(add(i + 2)); + EXPECT_EQ(firstBufferId, bufferId); } - TEST_DO(f.assertStoreContent()); + assertStoreContent(); - uint32_t bias = F1::EntryRefType::offsetSize(); - uint32_t secondBufferId = f.getBufferId(f.add(bias + 1)); - EXPECT_NOT_EQUAL(firstBufferId, secondBufferId); + uint32_t bias = SmallOffsetNumberTest::EntryRefType::offsetSize(); + uint32_t secondBufferId = getBufferId(add(bias + 1)); + EXPECT_NE(firstBufferId, secondBufferId); for (uint32_t i = 0; i < 10u; ++i) { - uint32_t bufferId = f.getBufferId(f.add(bias + i + 2)); - EXPECT_EQUAL(secondBufferId, bufferId); + uint32_t bufferId = getBufferId(add(bias + i + 2)); + EXPECT_EQ(secondBufferId, bufferId); } - TEST_DO(f.assertStoreContent()); + assertStoreContent(); } -TEST_F("require that compaction works", NumberFixture) +TEST_F(NumberTest, store_can_be_compacted) { - EntryRef val1Ref = f.add(1); - EntryRef val2Ref = f.add(2); - f.remove(f.add(4)); - f.trimHoldLists(); - TEST_DO(f.assertBufferState(val1Ref, MemStats().used(4).dead(2))); // Note: First element is reserved - uint32_t val1BufferId = f.getBufferId(val1Ref); + EntryRef val1Ref = add(1); + EntryRef val2Ref = add(2); + remove(add(4)); + trimHoldLists(); + assertBufferState(val1Ref, MemStats().used(4).dead(2)); // Note: First element is reserved + uint32_t val1BufferId = getBufferId(val1Ref); - EXPECT_EQUAL(2u, f.refStore.size()); - f.compactWorst(); - EXPECT_EQUAL(2u, f.refStore.size()); - TEST_DO(f.assertStoreContent()); + EXPECT_EQ(2u, refStore.size()); + compactWorst(); + EXPECT_EQ(2u, refStore.size()); + assertStoreContent(); // Buffer has been compacted - EXPECT_NOT_EQUAL(val1BufferId, f.getBufferId(f.getEntryRef(1))); + EXPECT_NE(val1BufferId, getBufferId(getEntryRef(1))); // Old ref should still point to data. - f.assertGet(val1Ref, 1); - f.assertGet(val2Ref, 2); - EXPECT_TRUE(f.store.bufferState(val1Ref).isOnHold()); - f.trimHoldLists(); - EXPECT_TRUE(f.store.bufferState(val1Ref).isFree()); - TEST_DO(f.assertStoreContent()); + assertGet(val1Ref, 1); + assertGet(val2Ref, 2); + EXPECT_TRUE(store.bufferState(val1Ref).isOnHold()); + trimHoldLists(); + EXPECT_TRUE(store.bufferState(val1Ref).isFree()); + assertStoreContent(); } -TEST_F("require that builder works", NumberFixture) +TEST_F(NumberTest, store_can_be_instantiated_with_builder) { - auto builder = f.getBuilder(2); + auto builder = getBuilder(2); builder.add(10); builder.add(20); builder.setupRefCounts(); EntryRef val10Ref = builder.mapEnumValueToEntryRef(1); EntryRef val20Ref = builder.mapEnumValueToEntryRef(2); - TEST_DO(f.assertBufferState(val10Ref, MemStats().used(3).dead(1))); // Note: First element is reserved + assertBufferState(val10Ref, MemStats().used(3).dead(1)); // Note: First element is reserved EXPECT_TRUE(val10Ref.valid()); EXPECT_TRUE(val20Ref.valid()); - EXPECT_NOT_EQUAL(val10Ref.ref(), val20Ref.ref()); - f.assertGet(val10Ref, 10); - f.assertGet(val20Ref, 20); + EXPECT_NE(val10Ref.ref(), val20Ref.ref()); + assertGet(val10Ref, 10); + assertGet(val20Ref, 20); builder.makeDictionary(); // Align refstore with the two entries added by builder. - f.alignRefStore(val10Ref, 10, 1); - f.alignRefStore(val20Ref, 20, 1); - EXPECT_EQUAL(val10Ref.ref(), f.add(10).ref()); - EXPECT_EQUAL(val20Ref.ref(), f.add(20).ref()); + alignRefStore(val10Ref, 10, 1); + alignRefStore(val20Ref, 20, 1); + EXPECT_EQ(val10Ref.ref(), add(10).ref()); + EXPECT_EQ(val20Ref.ref(), add(20).ref()); } -TEST_F("require that saver works", NumberFixture) +TEST_F(NumberTest, store_can_be_saved) { - EntryRef val10Ref = f.add(10); - EntryRef val20Ref = f.add(20); - f.remove(f.add(40)); - f.trimHoldLists(); + EntryRef val10Ref = add(10); + EntryRef val20Ref = add(20); + remove(add(40)); + trimHoldLists(); - auto saver = f.getSaver(); + auto saver = getSaver(); std::vector<uint32_t> refs; saver.foreach_key([&](EntryRef ref) { refs.push_back(ref.ref()); }); std::vector<uint32_t> expRefs; expRefs.push_back(val10Ref.ref()); expRefs.push_back(val20Ref.ref()); - EXPECT_EQUAL(expRefs, refs); + EXPECT_EQ(expRefs, refs); saver.enumerateValues(); uint32_t invalidEnum = saver.mapEntryRefToEnumValue(EntryRef()); uint32_t enumValue10 = saver.mapEntryRefToEnumValue(val10Ref); uint32_t enumValue20 = saver.mapEntryRefToEnumValue(val20Ref); - EXPECT_EQUAL(0u, invalidEnum); - EXPECT_EQUAL(1u, enumValue10); - EXPECT_EQUAL(2u, enumValue20); + EXPECT_EQ(0u, invalidEnum); + EXPECT_EQ(1u, enumValue10); + EXPECT_EQ(2u, enumValue20); } -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS() |