diff options
author | Ola Aunrønning <olaa@verizonmedia.com> | 2021-03-02 17:35:20 +0100 |
---|---|---|
committer | Ola Aunrønning <olaa@verizonmedia.com> | 2021-03-04 13:31:52 +0100 |
commit | 301f68c3b48b5ecbb94e0671fd710d0672afb046 (patch) | |
tree | 4b09f3fd36690c0ea4a90657dafd34b52bb4af1a | |
parent | 65b1933e6b2c1b5a2b2c678490590c2ad1af3cc2 (diff) |
Add cloud secret store config
10 files changed, 80 insertions, 17 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SecretStoreValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SecretStoreValidator.java index d26ff49c4fa..bafc0b8b3de 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SecretStoreValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SecretStoreValidator.java @@ -7,8 +7,11 @@ import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.IdentityProvider; +import com.yahoo.vespa.model.container.SecretStore; import com.yahoo.vespa.model.container.component.Component; +import java.util.List; + /** * Validates the requirements for setting up a secret store. * @@ -16,14 +19,23 @@ import com.yahoo.vespa.model.container.component.Component; */ public class SecretStoreValidator extends Validator { + private final List<String> VALID_TYPES = List.of("oath-ckms", "cloud"); + @Override public void validate(VespaModel model, DeployState deployState) { if (! deployState.isHosted()) return; if (model.getAdmin().getApplicationType() != ApplicationType.DEFAULT) return; for (ContainerCluster cluster : model.getContainerClusters().values()) { - if (cluster.getSecretStore().isPresent() && ! hasIdentityProvider(cluster)) { - throw new IllegalArgumentException(String.format( + + if (cluster.getSecretStore().isPresent() ) { + var secretStore = (SecretStore) cluster.getSecretStore().get(); + + if (!isValidType(secretStore.getType())) { + throw new IllegalArgumentException("Secret store must be one of the following types: " + VALID_TYPES); + } + if (! secretStore.isCloud() && ! hasIdentityProvider(cluster)) + throw new IllegalArgumentException(String.format( "Container cluster '%s' uses a secret store, so an Athenz domain and an Athenz service" + " must be declared in deployment.xml.", cluster.getName())); } @@ -36,4 +48,8 @@ public class SecretStoreValidator extends Validator { } return false; } + + private boolean isValidType(String type) { + return VALID_TYPES.contains(type); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index a1db3c43f1c..55d6c0ba6a9 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.container; import com.yahoo.cloud.config.ClusterInfoConfig; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.cloud.config.CuratorConfig; +import com.yahoo.cloud.config.SecretStoreConfig; import com.yahoo.component.ComponentId; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.docproc.DocprocConfig; @@ -101,7 +102,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> DocprocConfig.Producer, ClusterInfoConfig.Producer, ConfigserverConfig.Producer, - CuratorConfig.Producer + CuratorConfig.Producer, + SecretStoreConfig.Producer { /** @@ -577,6 +579,16 @@ public abstract class ContainerCluster<CONTAINER extends Container> builder.zookeeperLocalhostAffinity(zooKeeperLocalhostAffinity); } + @Override + public void getConfig(SecretStoreConfig.Builder builder) { + secretStore.getGroups().forEach(group -> + builder.groups(new SecretStoreConfig.Groups.Builder() + .name(group.name) + .region(group.environment) + ) + ); + } + private List<ClusterInfoConfig.Services.Ports.Builder> getPorts(Service service) { List<ClusterInfoConfig.Services.Ports.Builder> builders = new ArrayList<>(); PortsMeta portsMeta = service.getPortsMeta(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/SecretStore.java b/config-model/src/main/java/com/yahoo/vespa/model/container/SecretStore.java index c803168af81..0332192c5c6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/SecretStore.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/SecretStore.java @@ -11,6 +11,11 @@ import java.util.List; */ public class SecretStore { private final List<Group> groups = new ArrayList<>(); + private final String type; + + public SecretStore(String type) { + this.type = type; + } public void addGroup(String name, String environment) { groups.add(new Group(name, environment)); @@ -20,6 +25,14 @@ public class SecretStore { return ImmutableList.copyOf(groups); } + public String getType() { + return type; + } + + public boolean isCloud() { + return "cloud".equals(type); + } + public static class Group { public final String name; public final String environment; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index e7e82d577f4..4dc20fc668b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -29,7 +29,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.container.logging.FileConnectionLog; import com.yahoo.osgi.provider.model.ComponentModel; @@ -258,9 +257,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addSecretStore(ApplicationContainerCluster cluster, Element spec) { Element secretStoreElement = XML.getChild(spec, "secret-store"); if (secretStoreElement != null) { - SecretStore secretStore = new SecretStore(); + SecretStore secretStore = new SecretStore(secretStoreElement.getAttribute("type")); + String attributeName = secretStore.isCloud() ? "region" : "environment"; for (Element group : XML.getChildren(secretStoreElement, "group")) { - secretStore.addGroup(group.getAttribute("name"), group.getAttribute("environment")); + secretStore.addGroup(group.getAttribute("name"), group.getAttribute(attributeName)); } cluster.setSecretStore(secretStore); } diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 5257345c39e..abe7386fa00 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -86,10 +86,11 @@ AccessLog = element accesslog { } SecretStore = element secret-store { - attribute type { string "oath-ckms" } & + attribute type { string "oath-ckms" | string "cloud" } & element group { attribute name { string } & - attribute environment { string "alpha" | string "corp" | string "prod" | string "aws" | string "aws_stage" } + (attribute environment { string "alpha" | string "corp" | string "prod" | string "aws" | string "aws_stage" } | + attribute region { string } ) } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SecretStoreValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SecretStoreValidatorTest.java index da81c74fdd2..4a9535623bb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SecretStoreValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SecretStoreValidatorTest.java @@ -27,7 +27,7 @@ public class SecretStoreValidatorTest { private static String servicesXml() { return joinLines("<services version='1.0'>", " <container id='default' version='1.0'>", - " <secret-store>", + " <secret-store type='oath-ckms'>", " <group name='group1' environment='prod'/>", " </secret-store>", " </container>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index e0e4cbf19c0..92e0b116878 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -704,7 +704,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { public void secret_store_can_be_set_up() { Element clusterElem = DomBuilderTest.parse( "<container version='1.0'>", - " <secret-store>", + " <secret-store type='oath-ckms'>", " <group name='group1' environment='env1'/>", " </secret-store>", "</container>"); diff --git a/container-disc/src/main/resources/configdefinitions/cloud.config.secret-store.def b/container-disc/src/main/resources/configdefinitions/cloud.config.secret-store.def new file mode 100644 index 00000000000..04408441889 --- /dev/null +++ b/container-disc/src/main/resources/configdefinitions/cloud.config.secret-store.def @@ -0,0 +1,5 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config + +groups[].name string +groups[].region string
\ No newline at end of file diff --git a/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStore.java b/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStore.java index 4fbd42402d7..1636c6aeb6d 100644 --- a/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStore.java +++ b/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStore.java @@ -9,6 +9,7 @@ import com.amazonaws.services.simplesystemsmanagement.AWSSimpleSystemsManagement import com.amazonaws.services.simplesystemsmanagement.AWSSimpleSystemsManagementClient; import com.amazonaws.services.simplesystemsmanagement.model.GetParametersRequest; import com.amazonaws.services.simplesystemsmanagement.model.GetParametersResult; +import com.yahoo.cloud.config.SecretStoreConfig; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; @@ -20,18 +21,20 @@ public class AwsParameterStore implements SecretStore { private final VespaAwsCredentialsProvider credentialsProvider; private final String roleToAssume; private final String externalId; + private final String region; - AwsParameterStore(VespaAwsCredentialsProvider credentialsProvider, String roleToAssume, String externalId) { + AwsParameterStore(VespaAwsCredentialsProvider credentialsProvider, String roleToAssume, String externalId, String region) { this.credentialsProvider = credentialsProvider; this.roleToAssume = roleToAssume; this.externalId = externalId; + this.region = region; } @Override public String getSecret(String key) { AWSSecurityTokenService tokenService = AWSSecurityTokenServiceClientBuilder .standard() - .withRegion("us-east-1") + .withRegion(region) .withCredentials(credentialsProvider) .build(); @@ -43,7 +46,7 @@ public class AwsParameterStore implements SecretStore { AWSSimpleSystemsManagement client = AWSSimpleSystemsManagementClient.builder() .withCredentials(assumeExtAccountRole) - .withRegion("us-east-1") + .withRegion(region) .build(); GetParametersRequest parametersRequest = new GetParametersRequest().withNames(key).withWithDecryption(true); diff --git a/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStoreValidationHandler.java b/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStoreValidationHandler.java index 91b643066fb..5d5cad2f75d 100644 --- a/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStoreValidationHandler.java +++ b/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStoreValidationHandler.java @@ -2,6 +2,7 @@ package com.yahoo.jdisc.cloud.aws; import com.google.inject.Inject; +import com.yahoo.cloud.config.SecretStoreConfig; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; @@ -28,16 +29,18 @@ public class AwsParameterStoreValidationHandler extends LoggingRequestHandler { private static final Logger log = Logger.getLogger(AwsParameterStoreValidationHandler.class.getName()); private final VespaAwsCredentialsProvider credentialsProvider; + private final SecretStoreConfig secretStoreConfig; @Inject - public AwsParameterStoreValidationHandler(Context ctx) { - this(ctx, new VespaAwsCredentialsProvider()); + public AwsParameterStoreValidationHandler(Context ctx, SecretStoreConfig secretStoreConfig) { + this(ctx, secretStoreConfig, new VespaAwsCredentialsProvider()); } - public AwsParameterStoreValidationHandler(Context ctx, VespaAwsCredentialsProvider credentialsProvider) { + public AwsParameterStoreValidationHandler(Context ctx, SecretStoreConfig secretStoreConfig, VespaAwsCredentialsProvider credentialsProvider) { super(ctx); this.credentialsProvider = credentialsProvider; + this.secretStoreConfig = secretStoreConfig; } @@ -64,7 +67,8 @@ public class AwsParameterStoreValidationHandler extends LoggingRequestHandler { try { var arn = "arn:aws:iam::" + settings.awsId + ":role/" + settings.role; - var store = new AwsParameterStore(this.credentialsProvider, arn, settings.externalId); + var region = getRegion(settings); + var store = new AwsParameterStore(this.credentialsProvider, arn, settings.externalId, region); store.getSecret("vespa-secret"); root.setString("status", "ok"); } catch (RuntimeException e) { @@ -86,6 +90,15 @@ public class AwsParameterStoreValidationHandler extends LoggingRequestHandler { } } + private String getRegion(AwsSettings settings) { + return secretStoreConfig.groups() + .stream() + .filter(group -> group.name().equals(settings.name)) + .map(SecretStoreConfig.Groups::region) + .findFirst() + .orElseThrow(() -> new RuntimeException("No secret store named '" + settings.name + "' configured in services.xml")); + } + private static class AwsSettings { String name; String role; |