diff options
40 files changed, 636 insertions, 377 deletions
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java index 0eeeae457b6..7c25e906b6f 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java @@ -2,13 +2,17 @@ package com.yahoo.vespa.hosted.athenz.instanceproviderservice; import com.google.inject.Inject; -import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.Zone; -import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; +import com.yahoo.jdisc.http.ssl.impl.TlsContextBasedProvider; import com.yahoo.log.LogLevel; import com.yahoo.security.KeyStoreBuilder; import com.yahoo.security.KeyStoreType; import com.yahoo.security.KeyUtils; +import com.yahoo.security.SslContextBuilder; +import com.yahoo.security.tls.DefaultTlsContext; +import com.yahoo.security.tls.MutableX509KeyManager; +import com.yahoo.security.tls.PeerAuthentication; +import com.yahoo.security.tls.TlsContext; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; import com.yahoo.vespa.athenz.client.zts.Identity; @@ -17,12 +21,11 @@ import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.utils.SiaUtils; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; -import org.eclipse.jetty.util.ssl.SslContextFactory; +import javax.net.ssl.SSLContext; import java.net.URI; import java.nio.file.Path; import java.nio.file.Paths; -import java.security.GeneralSecurityException; import java.security.KeyPair; import java.security.KeyStore; import java.security.PrivateKey; @@ -42,14 +45,15 @@ import java.util.logging.Logger; * * @author bjorncs */ -public class ConfigserverSslContextFactoryProvider extends AbstractComponent implements SslContextFactoryProvider { +public class ConfigserverSslContextFactoryProvider extends TlsContextBasedProvider { private static final String CERTIFICATE_ALIAS = "athenz"; private static final Duration EXPIRATION_MARGIN = Duration.ofHours(6); private static final Path VESPA_SIA_DIRECTORY = Paths.get(Defaults.getDefaults().underVespaHome("var/vespa/sia")); private static final Logger log = Logger.getLogger(ConfigserverSslContextFactoryProvider.class.getName()); - private final SslContextFactory sslContextFactory; + private final TlsContext tlsContext; + private final MutableX509KeyManager keyManager = new MutableX509KeyManager(); private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(runnable -> new Thread(runnable, "configserver-ssl-context-factory-provider")); private final ZtsClient ztsClient; @@ -69,25 +73,20 @@ public class ConfigserverSslContextFactoryProvider extends AbstractComponent imp Duration updatePeriod = Duration.ofDays(config.updatePeriodDays()); Path trustStoreFile = Paths.get(config.athenzCaTrustStore()); - this.sslContextFactory = initializeSslContextFactory(keyProvider, trustStoreFile, updatePeriod, configserverIdentity, ztsClient, athenzProviderServiceConfig); - scheduler.scheduleAtFixedRate(new KeystoreUpdater(sslContextFactory), + this.tlsContext = createTlsContext(keyProvider, keyManager, trustStoreFile, updatePeriod, configserverIdentity, ztsClient, athenzProviderServiceConfig); + scheduler.scheduleAtFixedRate(new KeystoreUpdater(keyManager), updatePeriod.toDays()/*initial delay*/, updatePeriod.toDays(), TimeUnit.DAYS); } @Override - public SslContextFactory getInstance(String containerId, int port) { - return sslContextFactory; + protected TlsContext getTlsContext(String containerId, int port) { + return tlsContext; } Instant getCertificateNotAfter() { - try { - X509Certificate certificate = (X509Certificate) sslContextFactory.getKeyStore().getCertificate(CERTIFICATE_ALIAS); - return certificate.getNotAfter().toInstant(); - } catch (GeneralSecurityException e) { - throw new IllegalStateException("Unable to find configserver certificate from keystore: " + e.getMessage(), e); - } + return keyManager.getCertificateChain(CERTIFICATE_ALIAS)[0].getNotAfter().toInstant(); } @Override @@ -96,38 +95,28 @@ public class ConfigserverSslContextFactoryProvider extends AbstractComponent imp scheduler.shutdownNow(); scheduler.awaitTermination(30, TimeUnit.SECONDS); ztsClient.close(); + super.deconstruct(); } catch (InterruptedException e) { throw new RuntimeException("Failed to shutdown Athenz certificate updater on time", e); } } - private static SslContextFactory initializeSslContextFactory(KeyProvider keyProvider, - Path trustStoreFile, - Duration updatePeriod, - AthenzService configserverIdentity, - ZtsClient ztsClient, - AthenzProviderServiceConfig zoneConfig) { - - // TODO Use DefaultTlsContext to configure SslContextFactory (ensure that cipher/protocol configuration is same across all TLS endpoints) - - SslContextFactory.Server factory = new SslContextFactory.Server(); - - factory.setWantClientAuth(true); - - KeyStore trustStore = - KeyStoreBuilder.withType(KeyStoreType.JKS) - .fromFile(trustStoreFile) - .build(); - factory.setTrustStore(trustStore); - + private static TlsContext createTlsContext(KeyProvider keyProvider, + MutableX509KeyManager keyManager, + Path trustStoreFile, + Duration updatePeriod, + AthenzService configserverIdentity, + ZtsClient ztsClient, + AthenzProviderServiceConfig zoneConfig) { KeyStore keyStore = tryReadKeystoreFile(configserverIdentity, updatePeriod) .orElseGet(() -> updateKeystore(configserverIdentity, generateKeystorePassword(), keyProvider, ztsClient, zoneConfig)); - factory.setKeyStore(keyStore); - factory.setKeyStorePassword(""); - factory.setExcludeProtocols("TLSv1.3"); // TLSv1.3 is broken is multiple OpenJDK 11 versions - factory.setEndpointIdentificationAlgorithm(null); // disable https hostname verification of clients (must be disabled when using Athenz x509 certificates) - return factory; + keyManager.updateKeystore(keyStore, new char[0]); + SSLContext sslContext = new SslContextBuilder() + .withTrustStore(trustStoreFile) + .withKeyManager(keyManager) + .build(); + return new DefaultTlsContext(sslContext, PeerAuthentication.WANT); } private static Optional<KeyStore> tryReadKeystoreFile(AthenzService configserverIdentity, Duration updatePeriod) { @@ -171,10 +160,10 @@ public class ConfigserverSslContextFactoryProvider extends AbstractComponent imp } private class KeystoreUpdater implements Runnable { - final SslContextFactory sslContextFactory; + final MutableX509KeyManager keyManager; - KeystoreUpdater(SslContextFactory sslContextFactory) { - this.sslContextFactory = sslContextFactory; + KeystoreUpdater(MutableX509KeyManager keyManager) { + this.keyManager = keyManager; } @Override @@ -183,10 +172,7 @@ public class ConfigserverSslContextFactoryProvider extends AbstractComponent imp log.log(LogLevel.INFO, "Updating configserver provider certificate from ZTS"); char[] keystorePwd = generateKeystorePassword(); KeyStore keyStore = updateKeystore(configserverIdentity, keystorePwd, keyProvider, ztsClient, athenzProviderServiceConfig); - sslContextFactory.reload(scf -> { - scf.setKeyStore(keyStore); - scf.setKeyStorePassword(new String(keystorePwd)); - }); + keyManager.updateKeystore(keyStore, keystorePwd); log.log(LogLevel.INFO, "Certificate successfully updated"); } catch (Throwable t) { log.log(LogLevel.ERROR, "Failed to update certificate from ZTS: " + t.getMessage(), t); diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 9d7ae9759c3..3b4bc1c2c5c 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -57,8 +57,6 @@ public interface ModelContext { boolean useFdispatchByDefault(); boolean dispatchWithProtobuf(); boolean useAdaptiveDispatch(); - // TODO: Remove when 7.61 is the oldest model in use - default boolean enableMetricsProxyContainer() { return false; } // TODO: Remove temporary default implementation default Optional<TlsSecrets> tlsSecrets() { return Optional.empty(); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java index a7c0ebd4a07..a871da20669 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java @@ -28,12 +28,19 @@ public class ExactMatch extends Processor { @Override public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { - Matching.Type matching = field.getMatching().getType(); - if (matching.equals(Matching.Type.EXACT) || matching.equals(Matching.Type.WORD)) { - implementExactMatch(field, search); - } else if (field.getMatching().getExactMatchTerminator() != null) { - warn(search, field, "exact-terminator requires 'exact' matching to have any effect."); - } + processField(field, search); + } + } + + private void processField(SDField field, Search search) { + Matching.Type matching = field.getMatching().getType(); + if (matching.equals(Matching.Type.EXACT) || matching.equals(Matching.Type.WORD)) { + implementExactMatch(field, search); + } else if (field.getMatching().getExactMatchTerminator() != null) { + warn(search, field, "exact-terminator requires 'exact' matching to have any effect."); + } + for (var structField : field.getStructFields()) { + processField(structField, search); } } diff --git a/config-model/src/test/derived/exactmatch/exactmatch.sd b/config-model/src/test/derived/exactmatch/exactmatch.sd index d5104fbf36d..2f1d54c970a 100644 --- a/config-model/src/test/derived/exactmatch/exactmatch.sd +++ b/config-model/src/test/derived/exactmatch/exactmatch.sd @@ -3,6 +3,11 @@ search exactmatch { document exactmatch { + struct elem { + field name type string {} + field weight type int {} + } + field tag type string { indexing: summary | index match: exact @@ -16,6 +21,32 @@ search exactmatch { } } + field string_map type map<string, string> { + indexing: summary + struct-field key { + indexing: attribute + match { + exact + exact-terminator: "*!!!*" + } + } + } + + field elem_map type map<string, elem> { + indexing: summary + struct-field value.name { + indexing: attribute + match: exact + } + } + + field elem_array type array<elem> { + indexing: summary + struct-field name { + indexing: attribute + match: exact + } + } } } diff --git a/config-model/src/test/derived/exactmatch/ilscripts.cfg b/config-model/src/test/derived/exactmatch/ilscripts.cfg index 38c0978474f..5595f954c4a 100644 --- a/config-model/src/test/derived/exactmatch/ilscripts.cfg +++ b/config-model/src/test/derived/exactmatch/ilscripts.cfg @@ -3,5 +3,11 @@ fieldmatchmaxlength 1000000 ilscript[].doctype "exactmatch" ilscript[].docfield[] "tag" ilscript[].docfield[] "screweduserids" +ilscript[].docfield[] "string_map" +ilscript[].docfield[] "elem_map" +ilscript[].docfield[] "elem_array" ilscript[].content[] "clear_state | guard { input tag | exact | summary tag | index tag; }" ilscript[].content[] "clear_state | guard { input screweduserids | exact | index screweduserids | summary screweduserids | attribute screweduserids; }" +ilscript[].content[] "input elem_array | passthrough elem_array" +ilscript[].content[] "input elem_map | passthrough elem_map" +ilscript[].content[] "input string_map | passthrough string_map" diff --git a/config-model/src/test/derived/exactmatch/index-info.cfg b/config-model/src/test/derived/exactmatch/index-info.cfg index a17ff68642e..a4a193b1fcd 100644 --- a/config-model/src/test/derived/exactmatch/index-info.cfg +++ b/config-model/src/test/derived/exactmatch/index-info.cfg @@ -15,3 +15,47 @@ indexinfo[].command[].indexname "screweduserids" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "screweduserids" indexinfo[].command[].command "exact *!!!*" +indexinfo[].command[].indexname "string_map.key" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "string_map.key" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "string_map.key" +indexinfo[].command[].command "exact *!!!*" +indexinfo[].command[].indexname "string_map.value" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "string_map" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "string_map" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "elem_map.key" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "elem_map.value.name" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "elem_map.value.name" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "elem_map.value.name" +indexinfo[].command[].command "exact @@" +indexinfo[].command[].indexname "elem_map.value.weight" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "elem_map.value.weight" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "elem_map.value" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "elem_map" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "elem_map" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "elem_array.name" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "elem_array.name" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "elem_array.name" +indexinfo[].command[].command "exact @@" +indexinfo[].command[].indexname "elem_array.weight" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "elem_array.weight" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "elem_array" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "elem_array" +indexinfo[].command[].command "multivalue" diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java index cb381253f6a..caeff01f440 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java @@ -149,7 +149,7 @@ public class MultiTenantRpcAuthorizer implements RpcAuthorizer { if (filesOwnedByApplication.contains(requestedFile)) { return; // allowed to access } - throw new AuthorizationException("Peer is not allowed to access file " + requestedFile.value()); + throw new AuthorizationException(String.format("Peer is not allowed to access file %s. Peer is owned by %s", requestedFile.value(), peerOwner.toShortString())); default: throw new AuthorizationException(String.format("'%s' nodes are not allowed to access files", peerIdentity.nodeType())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index dce7d2e68ac..9c320df2f6c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -29,9 +29,7 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.NotExistsException; import com.yahoo.vespa.hosted.controller.api.ActivateResult; -import com.yahoo.vespa.hosted.controller.api.application.v4.ApplicationResource; import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource; -import com.yahoo.vespa.hosted.controller.api.application.v4.TenantResource; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RefeedAction; @@ -68,7 +66,6 @@ import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; import com.yahoo.vespa.hosted.controller.restapi.ResourceResponse; import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; -import com.yahoo.vespa.hosted.controller.restapi.StringResponse; import com.yahoo.vespa.hosted.controller.security.AccessControlRequests; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; @@ -267,7 +264,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse handleOPTIONS() { // We implement this to avoid redirect loops on OPTIONS requests from browsers, but do not really bother // spelling out the methods supported at each path, which we should - EmptyJsonResponse response = new EmptyJsonResponse(); + EmptyResponse response = new EmptyResponse(); response.headers().put("Allow", "GET,PUT,POST,PATCH,DELETE,OPTIONS"); return response; } @@ -945,12 +942,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Optional<Hostname> hostname = Optional.ofNullable(request.getProperty("hostname")).map(Hostname::new); controller.applications().restart(deploymentId, hostname); - // TODO: Change to return JSON - return new StringResponse("Requested restart of " + path(TenantResource.API_PATH, tenantName, - ApplicationResource.API_PATH, applicationName, - EnvironmentResource.API_PATH, environment, - "region", region, - "instance", instanceName)); + return new MessageResponse("Requested restart of " + deploymentId); } private HttpResponse jobDeploy(ApplicationId id, JobType type, HttpRequest request) { @@ -1097,21 +1089,17 @@ public class ApplicationApiHandler extends LoggingRequestHandler { ? Optional.empty() : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest())); controller.applications().deleteApplication(id, credentials); - return new EmptyJsonResponse(); // TODO: Replicates current behavior but should return a message response instead + return new MessageResponse("Deleted application " + id); } private HttpResponse deactivate(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { Application application = controller.applications().require(ApplicationId.from(tenantName, applicationName, instanceName)); // Attempt to deactivate application even if the deployment is not known by the controller - controller.applications().deactivate(application.id(), ZoneId.from(environment, region)); - - // TODO: Change to return JSON - return new StringResponse("Deactivated " + path(TenantResource.API_PATH, tenantName, - ApplicationResource.API_PATH, applicationName, - "instance", instanceName, - EnvironmentResource.API_PATH, environment, - "region", region)); + DeploymentId deploymentId = new DeploymentId(application.id(), ZoneId.from(environment, region)); + controller.applications().deactivate(deploymentId.applicationId(), deploymentId.zoneId()); + + return new MessageResponse("Deactivated " + deploymentId); } private HttpResponse notifyJobCompletion(String tenant, String application, HttpRequest request) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyJsonResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyResponse.java index be3222cc1a8..e343615f066 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyJsonResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyResponse.java @@ -8,16 +8,13 @@ import java.io.OutputStream; /** * @author bratseth */ -public class EmptyJsonResponse extends HttpResponse { +public class EmptyResponse extends HttpResponse { - public EmptyJsonResponse() { + public EmptyResponse() { super(200); } @Override public void render(OutputStream stream) {} - @Override - public String getContentType() { return "application/json"; } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 978b7e4397d..44b67a186b8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -8,18 +8,18 @@ import com.yahoo.config.provision.HostName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.restapi.Path; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.JobList; import com.yahoo.vespa.hosted.controller.application.JobStatus; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; import com.yahoo.vespa.hosted.controller.restapi.Uri; -import com.yahoo.vespa.hosted.controller.restapi.application.EmptyJsonResponse; -import com.yahoo.restapi.Path; +import com.yahoo.vespa.hosted.controller.restapi.application.EmptyResponse; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import java.util.Optional; @@ -71,7 +71,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler { private HttpResponse handleOPTIONS() { // We implement this to avoid redirect loops on OPTIONS requests from browsers, but do not really bother // spelling out the methods supported at each path, which we should - EmptyJsonResponse response = new EmptyJsonResponse(); + EmptyResponse response = new EmptyResponse(); response.headers().put("Allow", "GET,OPTIONS"); return response; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index 5ef997b6d55..7a76f13392d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -13,20 +13,19 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; import com.yahoo.vespa.hosted.controller.api.integration.user.UserId; import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; -import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; -import com.yahoo.vespa.hosted.controller.restapi.application.EmptyJsonResponse; +import com.yahoo.vespa.hosted.controller.restapi.application.EmptyResponse; import com.yahoo.yolean.Exceptions; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -99,7 +98,7 @@ public class UserApiHandler extends LoggingRequestHandler { } private HttpResponse handleOPTIONS() { - EmptyJsonResponse response = new EmptyJsonResponse(); + EmptyResponse response = new EmptyResponse(); response.headers().put("Allow", "GET,PUT,POST,PATCH,DELETE,OPTIONS"); return response; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java index d50d141d625..1ac82317695 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java @@ -2,15 +2,17 @@ package com.yahoo.vespa.hosted.controller.tls; import com.google.inject.Inject; -import com.yahoo.component.AbstractComponent; import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; +import com.yahoo.jdisc.http.ssl.impl.TlsContextBasedProvider; import com.yahoo.security.KeyStoreBuilder; import com.yahoo.security.KeyStoreType; import com.yahoo.security.KeyUtils; +import com.yahoo.security.SslContextBuilder; import com.yahoo.security.X509CertificateUtils; +import com.yahoo.security.tls.DefaultTlsContext; +import com.yahoo.security.tls.PeerAuthentication; +import com.yahoo.security.tls.TlsContext; import com.yahoo.vespa.hosted.controller.tls.config.TlsConfig; -import org.eclipse.jetty.util.ssl.SslContextFactory; import java.nio.file.Files; import java.nio.file.Paths; @@ -28,11 +30,11 @@ import java.util.concurrent.ConcurrentHashMap; * @author bjorncs */ @SuppressWarnings("unused") // Injected -public class ControllerSslContextFactoryProvider extends AbstractComponent implements SslContextFactoryProvider { +public class ControllerSslContextFactoryProvider extends TlsContextBasedProvider { private final KeyStore truststore; private final KeyStore keystore; - private final Map<Integer, SslContextFactory> sslContextFactories = new ConcurrentHashMap<>(); + private final Map<Integer, TlsContext> tlsContextMap = new ConcurrentHashMap<>(); @Inject public ControllerSslContextFactoryProvider(SecretStore secretStore, TlsConfig config) { @@ -50,24 +52,17 @@ public class ControllerSslContextFactoryProvider extends AbstractComponent imple } @Override - public SslContextFactory getInstance(String containerId, int port) { - return sslContextFactories.computeIfAbsent(port, this::createSslContextFactory); + protected TlsContext getTlsContext(String containerId, int port) { + return tlsContextMap.computeIfAbsent(port, this::createTlsContext); } - /** Create a SslContextFactory backed by an in-memory key and trust store */ - private SslContextFactory createSslContextFactory(int port) { - // TODO Use DefaultTlsContext to configure SslContextFactory (ensure that cipher/protocol configuration is same across all TLS endpoints). - - SslContextFactory.Server factory = new SslContextFactory.Server(); - if (port != 443) { - factory.setWantClientAuth(true); - } - factory.setTrustStore(truststore); - factory.setKeyStore(keystore); - factory.setKeyStorePassword(""); - factory.setExcludeProtocols("TLSv1.3"); // TLSv1.3 is broken is multiple OpenJDK 11 versions - factory.setEndpointIdentificationAlgorithm(null); // disable https hostname verification of clients (must be disabled when using Athenz x509 certificates) - return factory; + private TlsContext createTlsContext(int port) { + return new DefaultTlsContext( + new SslContextBuilder() + .withKeyStore(keystore, new char[0]) + .withTrustStore(truststore) + .build(), + port != 443 ? PeerAuthentication.WANT : PeerAuthentication.DISABLED); } /** Get private key from secret store **/ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index a0acbc625f7..29931a1f626 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -241,7 +241,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("deploy-result.json")); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/instance1", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/test/region/us-east-1"); + "{\"message\":\"Deactivated tenant1.application1.instance1 in test.us-east-1\"}"); controllerTester.jobCompletion(JobType.systemTest) .application(id) @@ -255,7 +255,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("deploy-result.json")); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/instance1", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/staging/region/us-east-3"); + "{\"message\":\"Deactivated tenant1.application1.instance1 in staging.us-east-3\"}"); controllerTester.jobCompletion(JobType.stagingTest) .application(id) .projectId(screwdriverProjectId) @@ -367,7 +367,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", DELETE) .userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), - ""); + "{\"message\":\"Deleted application tenant2.application2\"}"); // Set version 6.1 to broken to change compile version for. controllerTester.upgrader().overrideConfidence(Version.fromString("6.1"), VespaVersion.Confidence.broken); @@ -480,27 +480,27 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST a 'restart application' command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST) .userIdentity(USER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1"); + "{\"message\":\"Requested restart of tenant1.application1.instance1 in prod.us-central-1\"}"); // POST a 'restart application' command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST) .screwdriverIdentity(SCREWDRIVER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1"); + "{\"message\":\"Requested restart of tenant1.application1.instance1 in prod.us-central-1\"}"); // POST a 'restart application' in staging environment command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-central-1/instance/instance1/restart", POST) .screwdriverIdentity(SCREWDRIVER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/staging/region/us-central-1/instance/instance1"); + "{\"message\":\"Requested restart of tenant1.application1.instance1 in staging.us-central-1\"}"); // POST a 'restart application' in staging test command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-central-1/instance/instance1/restart", POST) .screwdriverIdentity(SCREWDRIVER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/test/region/us-central-1/instance/instance1"); + "{\"message\":\"Requested restart of tenant1.application1.instance1 in test.us-central-1\"}"); // POST a 'restart application' in staging dev command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-central-1/instance/instance1/restart", POST) .userIdentity(USER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/dev/region/us-central-1/instance/instance1"); + "{\"message\":\"Requested restart of tenant1.application1.instance1 in dev.us-central-1\"}"); // POST a 'restart application' command with a host filter (other filters not supported yet) tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart?hostname=host1", POST) @@ -530,18 +530,18 @@ public class ApplicationApiTest extends ControllerContainerTest { // DELETE (deactivate) a deployment - dev tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1", DELETE) .userIdentity(USER_ID), - "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/dev/region/us-west-1"); + "{\"message\":\"Deactivated tenant1.application1.instance1 in dev.us-west-1\"}"); // DELETE (deactivate) a deployment - prod tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1"); + "{\"message\":\"Deactivated tenant1.application1.instance1 in prod.us-central-1\"}"); // DELETE (deactivate) a deployment is idempotent tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1"); + "{\"message\":\"Deactivated tenant1.application1.instance1 in prod.us-central-1\"}"); // POST an application package to start a deployment to dev tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/dev-us-east-1", POST) @@ -661,7 +661,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // DELETE an application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", DELETE).userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), - ""); + "{\"message\":\"Deleted application tenant1.application1.instance1\"}"); // DELETE a tenant tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE).userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), @@ -990,7 +990,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", DELETE) .userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), - ""); + "{\"message\":\"Deleted application tenant1.application1.instance1\"}"); // DELETE application again - should produce 404 tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", DELETE) .oktaAccessToken(OKTA_AT) @@ -1089,7 +1089,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE) .userIdentity(authorizedUser) .oktaAccessToken(OKTA_AT), - "", + "{\"message\":\"Deleted application tenant1.application1\"}", 200); // Updating a tenant for an Athens domain the user is not admin for is disallowed @@ -1528,7 +1528,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("deploy-result.json")); tester.assertResponse(request(testPath, DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated " + testPath.replaceFirst("/application/v4/", "")); + "{\"message\":\"Deactivated " + application + " in test.us-east-1\"}"); controllerTester.jobCompletion(JobType.systemTest) .application(application) .projectId(projectId) @@ -1543,7 +1543,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("deploy-result.json")); tester.assertResponse(request(stagingPath, DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated " + stagingPath.replaceFirst("/application/v4/", "")); + "{\"message\":\"Deactivated " + application + " in staging.us-east-3\"}"); controllerTester.jobCompletion(JobType.stagingTest) .application(application) .projectId(projectId) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index 59f63f0472a..d0e9ae77965 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -137,7 +137,7 @@ public class UserApiTest extends ControllerContainerCloudTest { // DELETE an application is available to application admins. tester.assertResponse(request("/application/v4/tenant/my-tenant/application/my-app", DELETE) .roles(Set.of(Role.applicationAdmin(id.tenant(), id.application()))), - ""); + "{\"message\":\"Deleted application my-tenant.my-app\"}"); // DELETE a tenant role is available to tenant admins. tester.assertResponse(request("/user/v1/tenant/my-tenant", DELETE) diff --git a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp index 16005970817..dc39dfb04a7 100644 --- a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp +++ b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp @@ -355,7 +355,7 @@ struct TestContext { void test_tensor_reduce() { TEST_DO(test_reduce_op(Aggr::AVG, N())); TEST_DO(test_reduce_op(Aggr::COUNT, N())); - TEST_DO(test_reduce_op(Aggr::PROD, Sigmoid(N()))); + TEST_DO(test_reduce_op(Aggr::PROD, SigmoidF(N()))); TEST_DO(test_reduce_op(Aggr::SUM, N())); TEST_DO(test_reduce_op(Aggr::MAX, N())); TEST_DO(test_reduce_op(Aggr::MIN, N())); @@ -390,30 +390,30 @@ struct TestContext { } void test_tensor_map() { - TEST_DO(test_map_op("-a", operation::Neg::f, Sub2(Div10(N())))); + TEST_DO(test_map_op("-a", operation::Neg::f, Sub2(Div16(N())))); TEST_DO(test_map_op("!a", operation::Not::f, Mask2Seq(SkipNth(3)))); - TEST_DO(test_map_op("cos(a)", operation::Cos::f, Div10(N()))); - TEST_DO(test_map_op("sin(a)", operation::Sin::f, Div10(N()))); - TEST_DO(test_map_op("tan(a)", operation::Tan::f, Div10(N()))); - TEST_DO(test_map_op("cosh(a)", operation::Cosh::f, Div10(N()))); - TEST_DO(test_map_op("sinh(a)", operation::Sinh::f, Div10(N()))); - TEST_DO(test_map_op("tanh(a)", operation::Tanh::f, Div10(N()))); - TEST_DO(test_map_op("acos(a)", operation::Acos::f, Sigmoid(Div10(N())))); - TEST_DO(test_map_op("asin(a)", operation::Asin::f, Sigmoid(Div10(N())))); - TEST_DO(test_map_op("atan(a)", operation::Atan::f, Div10(N()))); - TEST_DO(test_map_op("exp(a)", operation::Exp::f, Div10(N()))); - TEST_DO(test_map_op("log10(a)", operation::Log10::f, Div10(N()))); - TEST_DO(test_map_op("log(a)", operation::Log::f, Div10(N()))); - TEST_DO(test_map_op("sqrt(a)", operation::Sqrt::f, Div10(N()))); - TEST_DO(test_map_op("ceil(a)", operation::Ceil::f, Div10(N()))); - TEST_DO(test_map_op("fabs(a)", operation::Fabs::f, Div10(N()))); - TEST_DO(test_map_op("floor(a)", operation::Floor::f, Div10(N()))); + TEST_DO(test_map_op("cos(a)", operation::Cos::f, Div16(N()))); + TEST_DO(test_map_op("sin(a)", operation::Sin::f, Div16(N()))); + TEST_DO(test_map_op("tan(a)", operation::Tan::f, Div16(N()))); + TEST_DO(test_map_op("cosh(a)", operation::Cosh::f, Div16(N()))); + TEST_DO(test_map_op("sinh(a)", operation::Sinh::f, Div16(N()))); + TEST_DO(test_map_op("tanh(a)", operation::Tanh::f, Div16(N()))); + TEST_DO(test_map_op("acos(a)", operation::Acos::f, SigmoidF(Div16(N())))); + TEST_DO(test_map_op("asin(a)", operation::Asin::f, SigmoidF(Div16(N())))); + TEST_DO(test_map_op("atan(a)", operation::Atan::f, Div16(N()))); + TEST_DO(test_map_op("exp(a)", operation::Exp::f, Div16(N()))); + TEST_DO(test_map_op("log10(a)", operation::Log10::f, Div16(N()))); + TEST_DO(test_map_op("log(a)", operation::Log::f, Div16(N()))); + TEST_DO(test_map_op("sqrt(a)", operation::Sqrt::f, Div16(N()))); + TEST_DO(test_map_op("ceil(a)", operation::Ceil::f, Div16(N()))); + TEST_DO(test_map_op("fabs(a)", operation::Fabs::f, Div16(N()))); + TEST_DO(test_map_op("floor(a)", operation::Floor::f, Div16(N()))); TEST_DO(test_map_op("isNan(a)", operation::IsNan::f, Mask2Seq(SkipNth(3), 1.0, my_nan))); - TEST_DO(test_map_op("relu(a)", operation::Relu::f, Sub2(Div10(N())))); - TEST_DO(test_map_op("sigmoid(a)", operation::Sigmoid::f, Sub2(Div10(N())))); - TEST_DO(test_map_op("elu(a)", operation::Elu::f, Sub2(Div10(N())))); + TEST_DO(test_map_op("relu(a)", operation::Relu::f, Sub2(Div16(N())))); + TEST_DO(test_map_op("sigmoid(a)", operation::Sigmoid::f, Sub2(Div16(N())))); + TEST_DO(test_map_op("elu(a)", operation::Elu::f, Sub2(Div16(N())))); TEST_DO(test_map_op("a in [1,5,7,13,42]", MyIn::f, N())); - TEST_DO(test_map_op("(a+1)*2", MyOp::f, Div10(N()))); + TEST_DO(test_map_op("(a+1)*2", MyOp::f, Div16(N()))); } //------------------------------------------------------------------------- @@ -666,27 +666,27 @@ struct TestContext { } void test_tensor_apply() { - TEST_DO(test_apply_op("a+b", operation::Add::f, Div10(N()))); - TEST_DO(test_apply_op("a-b", operation::Sub::f, Div10(N()))); - TEST_DO(test_apply_op("a*b", operation::Mul::f, Div10(N()))); - TEST_DO(test_apply_op("a/b", operation::Div::f, Div10(N()))); - TEST_DO(test_apply_op("a%b", operation::Mod::f, Div10(N()))); - TEST_DO(test_apply_op("a^b", operation::Pow::f, Div10(N()))); - TEST_DO(test_apply_op("pow(a,b)", operation::Pow::f, Div10(N()))); - TEST_DO(test_apply_op("a==b", operation::Equal::f, Div10(N()))); - TEST_DO(test_apply_op("a!=b", operation::NotEqual::f, Div10(N()))); - TEST_DO(test_apply_op("a~=b", operation::Approx::f, Div10(N()))); - TEST_DO(test_apply_op("a<b", operation::Less::f, Div10(N()))); - TEST_DO(test_apply_op("a<=b", operation::LessEqual::f, Div10(N()))); - TEST_DO(test_apply_op("a>b", operation::Greater::f, Div10(N()))); - TEST_DO(test_apply_op("a>=b", operation::GreaterEqual::f, Div10(N()))); + TEST_DO(test_apply_op("a+b", operation::Add::f, Div16(N()))); + TEST_DO(test_apply_op("a-b", operation::Sub::f, Div16(N()))); + TEST_DO(test_apply_op("a*b", operation::Mul::f, Div16(N()))); + TEST_DO(test_apply_op("a/b", operation::Div::f, Div16(N()))); + TEST_DO(test_apply_op("a%b", operation::Mod::f, Div16(N()))); + TEST_DO(test_apply_op("a^b", operation::Pow::f, Div16(N()))); + TEST_DO(test_apply_op("pow(a,b)", operation::Pow::f, Div16(N()))); + TEST_DO(test_apply_op("a==b", operation::Equal::f, Div16(N()))); + TEST_DO(test_apply_op("a!=b", operation::NotEqual::f, Div16(N()))); + TEST_DO(test_apply_op("a~=b", operation::Approx::f, Div16(N()))); + TEST_DO(test_apply_op("a<b", operation::Less::f, Div16(N()))); + TEST_DO(test_apply_op("a<=b", operation::LessEqual::f, Div16(N()))); + TEST_DO(test_apply_op("a>b", operation::Greater::f, Div16(N()))); + TEST_DO(test_apply_op("a>=b", operation::GreaterEqual::f, Div16(N()))); TEST_DO(test_apply_op("a&&b", operation::And::f, Mask2Seq(SkipNth(3)))); TEST_DO(test_apply_op("a||b", operation::Or::f, Mask2Seq(SkipNth(3)))); - TEST_DO(test_apply_op("atan2(a,b)", operation::Atan2::f, Div10(N()))); - TEST_DO(test_apply_op("ldexp(a,b)", operation::Ldexp::f, Div10(N()))); - TEST_DO(test_apply_op("fmod(a,b)", operation::Mod::f, Div10(N()))); - TEST_DO(test_apply_op("min(a,b)", operation::Min::f, Div10(N()))); - TEST_DO(test_apply_op("max(a,b)", operation::Max::f, Div10(N()))); + TEST_DO(test_apply_op("atan2(a,b)", operation::Atan2::f, Div16(N()))); + TEST_DO(test_apply_op("ldexp(a,b)", operation::Ldexp::f, Div16(N()))); + TEST_DO(test_apply_op("fmod(a,b)", operation::Mod::f, Div16(N()))); + TEST_DO(test_apply_op("min(a,b)", operation::Min::f, Div16(N()))); + TEST_DO(test_apply_op("max(a,b)", operation::Max::f, Div16(N()))); } //------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/eval/test/tensor_model.hpp b/eval/src/vespa/eval/eval/test/tensor_model.hpp index 4fad2820cf7..6efb7470d55 100644 --- a/eval/src/vespa/eval/eval/test/tensor_model.hpp +++ b/eval/src/vespa/eval/eval/test/tensor_model.hpp @@ -32,6 +32,14 @@ struct Div10 : Sequence { double operator[](size_t i) const override { return (seq[i] / 10.0); } }; +// Sequence of another sequence divided by 10 +struct Div16 : Sequence { + const Sequence &seq; + Div16(const Sequence &seq_in) : seq(seq_in) {} + double operator[](size_t i) const override { return (seq[i] / 16.0); } +}; + + // Sequence of another sequence minus 2 struct Sub2 : Sequence { const Sequence &seq; @@ -54,6 +62,13 @@ struct Sigmoid : Sequence { double operator[](size_t i) const override { return operation::Sigmoid::f(seq[i]); } }; +// Sequence of applying sigmoid to another sequence, plus rounding to nearest float +struct SigmoidF : Sequence { + const Sequence &seq; + SigmoidF(const Sequence &seq_in) : seq(seq_in) {} + double operator[](size_t i) const override { return (float)operation::Sigmoid::f(seq[i]); } +}; + // pre-defined sequence of numbers struct Seq : Sequence { std::vector<double> seq; diff --git a/jdisc_http_service/abi-spec.json b/jdisc_http_service/abi-spec.json index a326b5792be..f915dc1e8c1 100644 --- a/jdisc_http_service/abi-spec.json +++ b/jdisc_http_service/abi-spec.json @@ -1124,14 +1124,17 @@ }, "com.yahoo.jdisc.http.ssl.SslContextFactoryProvider": { "superClass": "java.lang.Object", - "interfaces": [], + "interfaces": [ + "java.lang.AutoCloseable" + ], "attributes": [ "public", "interface", "abstract" ], "methods": [ - "public abstract org.eclipse.jetty.util.ssl.SslContextFactory getInstance(java.lang.String, int)" + "public abstract org.eclipse.jetty.util.ssl.SslContextFactory getInstance(java.lang.String, int)", + "public void close()" ], "fields": [] } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslContextFactoryProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslContextFactoryProvider.java index 37916fd5734..c364116e0af 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslContextFactoryProvider.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslContextFactoryProvider.java @@ -8,7 +8,7 @@ import org.eclipse.jetty.util.ssl.SslContextFactory; * * @author bjorncs */ -public interface SslContextFactoryProvider { +public interface SslContextFactoryProvider extends AutoCloseable { /** * This method is called once for each SSL connector. @@ -17,4 +17,5 @@ public interface SslContextFactoryProvider { */ SslContextFactory getInstance(String containerId, int port); + @Override default void close() {} } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java index 0bbe6207294..615cd5d46ad 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java @@ -3,7 +3,7 @@ package com.yahoo.jdisc.http.ssl.impl; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; -import com.yahoo.security.tls.ReloadingTlsContext; +import com.yahoo.security.tls.ConfigFileBasedTlsContext; import com.yahoo.security.tls.TlsContext; import com.yahoo.security.tls.TransportSecurityUtils; import org.eclipse.jetty.util.ssl.SslContextFactory; @@ -15,21 +15,38 @@ import org.eclipse.jetty.util.ssl.SslContextFactory; */ public class DefaultSslContextFactoryProvider extends AbstractComponent implements SslContextFactoryProvider { - private final TlsContext tlsContext = TransportSecurityUtils.getConfigFile() - .map(configFile -> new ReloadingTlsContext(configFile, TransportSecurityUtils.getInsecureAuthorizationMode())) - .orElse(null); + private final SslContextFactoryProvider instance = TransportSecurityUtils.getConfigFile() + .map(configFile -> (SslContextFactoryProvider) new StaticTlsContextBasedProvider( + new ConfigFileBasedTlsContext(configFile, TransportSecurityUtils.getInsecureAuthorizationMode()))) + .orElseGet(ThrowingSslContextFactoryProvider::new); @Override public SslContextFactory getInstance(String containerId, int port) { - if (tlsContext != null) { - return new TlsContextManagedSslContextFactory(tlsContext); - } else { - throw new UnsupportedOperationException(); - } + return instance.getInstance(containerId, port); } @Override public void deconstruct() { - if (tlsContext != null) tlsContext.close(); + instance.close(); + } + + private static class ThrowingSslContextFactoryProvider implements SslContextFactoryProvider { + @Override + public SslContextFactory getInstance(String containerId, int port) { + throw new UnsupportedOperationException(); + } + } + + private static class StaticTlsContextBasedProvider extends TlsContextBasedProvider { + final TlsContext tlsContext; + + StaticTlsContextBasedProvider(TlsContext tlsContext) { + this.tlsContext = tlsContext; + } + + @Override + protected TlsContext getTlsContext(String containerId, int port) { + return tlsContext; + } } }
\ No newline at end of file diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java new file mode 100644 index 00000000000..e8ae13e48be --- /dev/null +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java @@ -0,0 +1,54 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.ssl.impl; + +import com.yahoo.component.AbstractComponent; +import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; +import com.yahoo.security.tls.TlsContext; +import org.eclipse.jetty.util.ssl.SslContextFactory; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLParameters; +import java.util.Arrays; + +/** + * A {@link SslContextFactoryProvider} that creates {@link SslContextFactory} instances from {@link TlsContext} instances. + * + * @author bjorncs + */ +public abstract class TlsContextBasedProvider extends AbstractComponent implements SslContextFactoryProvider { + + protected abstract TlsContext getTlsContext(String containerId, int port); + + @Override + public final SslContextFactory getInstance(String containerId, int port) { + TlsContext tlsContext = getTlsContext(containerId, port); + SSLContext sslContext = tlsContext.context(); + SSLParameters parameters = tlsContext.parameters(); + + SslContextFactory.Server sslContextFactory = new SslContextFactory.Server(); + sslContextFactory.setSslContext(sslContext); + + sslContextFactory.setNeedClientAuth(parameters.getNeedClientAuth()); + sslContextFactory.setWantClientAuth(parameters.getWantClientAuth()); + + String[] enabledProtocols = parameters.getProtocols(); + sslContextFactory.setIncludeProtocols(enabledProtocols); + String[] supportedProtocols = sslContext.getSupportedSSLParameters().getProtocols(); + sslContextFactory.setExcludeProtocols(createExclusionList(enabledProtocols, supportedProtocols)); + + String[] enabledCiphers = parameters.getCipherSuites(); + String[] supportedCiphers = sslContext.getSupportedSSLParameters().getCipherSuites(); + sslContextFactory.setIncludeCipherSuites(enabledCiphers); + sslContextFactory.setExcludeCipherSuites(createExclusionList(enabledCiphers, supportedCiphers)); + return sslContextFactory; + } + + private static String[] createExclusionList(String[] enabledValues, String[] supportedValues) { + return Arrays.stream(supportedValues) + .filter(supportedValue -> + Arrays.stream(enabledValues) + .noneMatch(enabledValue -> enabledValue.equals(supportedValue))) + .toArray(String[]::new); + } + +} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextManagedSslContextFactory.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextManagedSslContextFactory.java deleted file mode 100644 index a5652042f9e..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextManagedSslContextFactory.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.ssl.impl; - -import com.yahoo.security.tls.TlsContext; -import org.eclipse.jetty.util.ssl.SslContextFactory; - -import javax.net.ssl.SSLEngine; -import java.net.InetSocketAddress; - -/** - * A Jetty {@link SslContextFactory} backed by {@link TlsContext}. - * Overrides methods that are used by Jetty to construct ssl sockets and ssl engines. - * - * @author bjorncs - */ -class TlsContextManagedSslContextFactory extends SslContextFactory.Server { - - private final TlsContext tlsContext; - - TlsContextManagedSslContextFactory(TlsContext tlsContext) { - this.tlsContext = tlsContext; - } - - @Override protected void doStart() { } // Override default behaviour - @Override protected void doStop() { } // Override default behaviour - - @Override - public SSLEngine newSSLEngine() { - return tlsContext.createSslEngine(); - } - - @Override - public SSLEngine newSSLEngine(InetSocketAddress address) { - return tlsContext.createSslEngine(address.getHostString(), address.getPort()); - } - - @Override - public SSLEngine newSSLEngine(String host, int port) { - return tlsContext.createSslEngine(host, port); - } - -} diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java new file mode 100644 index 00000000000..88db5c99de9 --- /dev/null +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java @@ -0,0 +1,70 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.ssl.impl; + +import com.yahoo.security.KeyUtils; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.tls.AuthorizationMode; +import com.yahoo.security.tls.DefaultTlsContext; +import com.yahoo.security.tls.PeerAuthentication; +import com.yahoo.security.tls.TlsContext; +import com.yahoo.security.tls.policy.AuthorizedPeers; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.junit.Test; + +import javax.security.auth.x500.X500Principal; +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Set; + +import static com.yahoo.security.KeyAlgorithm.EC; +import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertNotNull; + +/** + * @author bjorncs + */ +public class TlsContextBasedProviderTest { + + @Test + public void creates_sslcontextfactory_from_tlscontext() { + TlsContext tlsContext = createTlsContext(); + var provider = new SimpleTlsContextBasedProvider(tlsContext); + SslContextFactory sslContextFactory = provider.getInstance("dummyContainerId", 8080); + assertNotNull(sslContextFactory); + assertArrayEquals(tlsContext.parameters().getCipherSuites(), sslContextFactory.getIncludeCipherSuites()); + } + + private static TlsContext createTlsContext() { + KeyPair keyPair = KeyUtils.generateKeypair(EC); + X509Certificate certificate = X509CertificateBuilder + .fromKeypair( + keyPair, + new X500Principal("CN=dummy"), + Instant.EPOCH, + Instant.EPOCH.plus(100000, ChronoUnit.DAYS), + SHA256_WITH_ECDSA, + BigInteger.ONE) + .build(); + return new DefaultTlsContext( + List.of(certificate), keyPair.getPrivate(), List.of(certificate), new AuthorizedPeers(Set.of()), AuthorizationMode.ENFORCE, PeerAuthentication.NEED); + } + + private static class SimpleTlsContextBasedProvider extends TlsContextBasedProvider { + final TlsContext tlsContext; + + SimpleTlsContextBasedProvider(TlsContext tlsContext) { + this.tlsContext = tlsContext; + } + + @Override + protected TlsContext getTlsContext(String containerId, int port) { + return tlsContext; + } + + } +}
\ No newline at end of file diff --git a/jrt/src/com/yahoo/jrt/CryptoEngine.java b/jrt/src/com/yahoo/jrt/CryptoEngine.java index 81bf10be187..8812264a3f1 100644 --- a/jrt/src/com/yahoo/jrt/CryptoEngine.java +++ b/jrt/src/com/yahoo/jrt/CryptoEngine.java @@ -4,7 +4,7 @@ package com.yahoo.jrt; import com.yahoo.security.tls.AuthorizationMode; import com.yahoo.security.tls.MixedMode; -import com.yahoo.security.tls.ReloadingTlsContext; +import com.yahoo.security.tls.ConfigFileBasedTlsContext; import com.yahoo.security.tls.TlsContext; import com.yahoo.security.tls.TransportSecurityUtils; @@ -24,7 +24,7 @@ public interface CryptoEngine extends AutoCloseable { return new NullCryptoEngine(); } AuthorizationMode mode = TransportSecurityUtils.getInsecureAuthorizationMode(); - TlsContext tlsContext = new ReloadingTlsContext(TransportSecurityUtils.getConfigFile().get(), mode); + TlsContext tlsContext = new ConfigFileBasedTlsContext(TransportSecurityUtils.getConfigFile().get(), mode); TlsCryptoEngine tlsCryptoEngine = new TlsCryptoEngine(tlsContext); MixedMode mixedMode = TransportSecurityUtils.getInsecureMixedMode(); switch (mixedMode) { diff --git a/jrt/tests/com/yahoo/jrt/CryptoUtils.java b/jrt/tests/com/yahoo/jrt/CryptoUtils.java index 6890fe88da5..e7e4eea568d 100644 --- a/jrt/tests/com/yahoo/jrt/CryptoUtils.java +++ b/jrt/tests/com/yahoo/jrt/CryptoUtils.java @@ -5,6 +5,7 @@ import com.yahoo.security.KeyUtils; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.tls.AuthorizationMode; import com.yahoo.security.tls.DefaultTlsContext; +import com.yahoo.security.tls.PeerAuthentication; import com.yahoo.security.tls.TlsContext; import com.yahoo.security.tls.policy.AuthorizedPeers; import com.yahoo.security.tls.policy.HostGlobPattern; @@ -48,7 +49,7 @@ class CryptoUtils { Field.CN, new HostGlobPattern("dummy")))))); static TlsContext createTestTlsContext() { - return new DefaultTlsContext(singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers, AuthorizationMode.ENFORCE, DefaultTlsContext.ALLOWED_CIPHER_SUITES); + return new DefaultTlsContext(singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers, AuthorizationMode.ENFORCE, PeerAuthentication.NEED); } } diff --git a/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java b/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java index 0ef179f775e..4f8919cdd5e 100644 --- a/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java +++ b/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java @@ -33,6 +33,7 @@ public class SslContextBuilder { private char[] keyStorePassword; private TrustManagerFactory trustManagerFactory = TrustManagerUtils::createDefaultX509TrustManager; private KeyManagerFactory keyManagerFactory = KeyManagerUtils::createDefaultX509KeyManager; + private X509ExtendedKeyManager keyManager; public SslContextBuilder() {} @@ -110,11 +111,23 @@ public class SslContextBuilder { return this; } + /** + * Note: Callee is responsible for configuring the key manager. + * Any keystore configured by {@link #withKeyStore(KeyStore, char[])} or the other overloads will be ignored. + */ + public SslContextBuilder withKeyManager(X509ExtendedKeyManager keyManager) { + this.keyManager = keyManager; + return this; + } + public SSLContext build() { try { SSLContext sslContext = SSLContext.getInstance("TLSv1.2"); TrustManager[] trustManagers = new TrustManager[] { trustManagerFactory.createTrustManager(trustStoreSupplier.get()) }; - KeyManager[] keyManagers = new KeyManager[] { keyManagerFactory.createKeyManager(keyStoreSupplier.get(), keyStorePassword) }; + X509ExtendedKeyManager keyManager = this.keyManager != null + ? this.keyManager + : keyManagerFactory.createKeyManager(keyStoreSupplier.get(), keyStorePassword); + KeyManager[] keyManagers = new KeyManager[] {keyManager}; sslContext.init(keyManagers, trustManagers, null); return sslContext; } catch (GeneralSecurityException e) { diff --git a/security-utils/src/main/java/com/yahoo/security/tls/AutoReloadingX509KeyManager.java b/security-utils/src/main/java/com/yahoo/security/tls/AutoReloadingX509KeyManager.java index 0dae185995c..faf6ecb4348 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/AutoReloadingX509KeyManager.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/AutoReloadingX509KeyManager.java @@ -31,6 +31,8 @@ import java.util.logging.Logger; */ public class AutoReloadingX509KeyManager extends X509ExtendedKeyManager implements AutoCloseable { + public static final String CERTIFICATE_ALIAS = "default"; + private static final Duration UPDATE_PERIOD = Duration.ofHours(1); private static final Logger log = Logger.getLogger(AutoReloadingX509KeyManager.class.getName()); @@ -61,7 +63,7 @@ public class AutoReloadingX509KeyManager extends X509ExtendedKeyManager implemen try { return KeyStoreBuilder.withType(KeyStoreType.PKCS12) .withKeyEntry( - "default", + CERTIFICATE_ALIAS, KeyUtils.fromPemEncodedPrivateKey(Files.readString(privateKey)), X509CertificateUtils.certificateListFromPem(Files.readString(certificateChain))) .build(); diff --git a/security-utils/src/main/java/com/yahoo/security/tls/ReloadingTlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java index 16f66f91da6..3b9158cf9b1 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/ReloadingTlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java @@ -20,6 +20,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.security.KeyStore; import java.time.Duration; +import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -29,20 +31,21 @@ import java.util.logging.Level; import java.util.logging.Logger; /** - * A {@link TlsContext} that regularly reloads the credentials referred to from the transport security options file. + * A {@link TlsContext} that uses the tls configuration specified in the transport security options file. + * The credentials are regularly reloaded to support short-lived certificates. * * @author bjorncs */ -public class ReloadingTlsContext implements TlsContext { +public class ConfigFileBasedTlsContext implements TlsContext { private static final Duration UPDATE_PERIOD = Duration.ofHours(1); - private static final Logger log = Logger.getLogger(ReloadingTlsContext.class.getName()); + private static final Logger log = Logger.getLogger(ConfigFileBasedTlsContext.class.getName()); private final TlsContext tlsContext; private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new ReloaderThreadFactory()); - public ReloadingTlsContext(Path tlsOptionsConfigFile, AuthorizationMode mode) { + public ConfigFileBasedTlsContext(Path tlsOptionsConfigFile, AuthorizationMode mode) { TransportSecurityOptions options = TransportSecurityOptions.fromJsonFile(tlsOptionsConfigFile); MutableX509TrustManager trustManager = new MutableX509TrustManager(); MutableX509KeyManager keyManager = new MutableX509KeyManager(); @@ -99,13 +102,15 @@ public class ReloadingTlsContext implements TlsContext { MutableX509TrustManager mutableTrustManager, MutableX509KeyManager mutableKeyManager) { SSLContext sslContext = new SslContextBuilder() - .withKeyManagerFactory((ignoredKeystore, ignoredPassword) -> mutableKeyManager) + .withKeyManager(mutableKeyManager) .withTrustManagerFactory( ignoredTruststore -> options.getAuthorizedPeers() .map(authorizedPeers -> (X509ExtendedTrustManager) new PeerAuthorizerTrustManager(authorizedPeers, mode, mutableTrustManager)) .orElseGet(() -> new PeerAuthorizerTrustManager(new AuthorizedPeers(Set.of()), AuthorizationMode.DISABLE, mutableTrustManager))) .build(); - return new DefaultTlsContext(sslContext, options.getAcceptedCiphers()); + List<String> acceptedCiphers = options.getAcceptedCiphers(); + Set<String> ciphers = acceptedCiphers.isEmpty() ? TlsContext.ALLOWED_CIPHER_SUITES : new HashSet<>(acceptedCiphers); + return new DefaultTlsContext(sslContext, ciphers, PeerAuthentication.NEED); } // Wrapped methods from TlsContext diff --git a/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java index e74ad49b2f5..572461c6cdd 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java @@ -23,47 +23,41 @@ import java.util.logging.Logger; */ public class DefaultTlsContext implements TlsContext { - public static final List<String> ALLOWED_CIPHER_SUITES = Arrays.asList( - "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", - "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", - "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", - "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", - "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", - "TLS_AES_128_GCM_SHA256", // TLSv1.3 - "TLS_AES_256_GCM_SHA384", // TLSv1.3 - "TLS_CHACHA20_POLY1305_SHA256"); // TLSv1.3 - - public static final List<String> ALLOWED_PROTOCOLS = List.of("TLSv1.2"); // TODO Enable TLSv1.3 - private static final Logger log = Logger.getLogger(DefaultTlsContext.class.getName()); private final SSLContext sslContext; private final String[] validCiphers; private final String[] validProtocols; + private final PeerAuthentication peerAuthentication; public DefaultTlsContext(List<X509Certificate> certificates, PrivateKey privateKey, List<X509Certificate> caCertificates, AuthorizedPeers authorizedPeers, AuthorizationMode mode, - List<String> acceptedCiphers) { - this(createSslContext(certificates, privateKey, caCertificates, authorizedPeers, mode), - acceptedCiphers); + PeerAuthentication peerAuthentication) { + this(createSslContext(certificates, privateKey, caCertificates, authorizedPeers, mode), peerAuthentication); + } + + public DefaultTlsContext(SSLContext sslContext, PeerAuthentication peerAuthentication) { + this(sslContext, TlsContext.ALLOWED_CIPHER_SUITES, peerAuthentication); } + public DefaultTlsContext(SSLContext sslContext) { + this(sslContext, TlsContext.ALLOWED_CIPHER_SUITES, PeerAuthentication.NEED); + } - public DefaultTlsContext(SSLContext sslContext, List<String> acceptedCiphers) { + DefaultTlsContext(SSLContext sslContext, Set<String> acceptedCiphers, PeerAuthentication peerAuthentication) { this.sslContext = sslContext; + this.peerAuthentication = peerAuthentication; this.validCiphers = getAllowedCiphers(sslContext, acceptedCiphers); this.validProtocols = getAllowedProtocols(sslContext); } - - private static String[] getAllowedCiphers(SSLContext sslContext, List<String> acceptedCiphers) { + private static String[] getAllowedCiphers(SSLContext sslContext, Set<String> acceptedCiphers) { String[] supportedCipherSuites = sslContext.getSupportedSSLParameters().getCipherSuites(); String[] validCipherSuites = Arrays.stream(supportedCipherSuites) - .filter(suite -> ALLOWED_CIPHER_SUITES.contains(suite) && (acceptedCiphers.isEmpty() || acceptedCiphers.contains(suite))) + .filter(suite -> ALLOWED_CIPHER_SUITES.contains(suite) && acceptedCiphers.contains(suite)) .toArray(String[]::new); if (validCipherSuites.length == 0) { throw new IllegalStateException( @@ -117,7 +111,18 @@ public class DefaultTlsContext implements TlsContext { SSLParameters newParameters = sslContext.getDefaultSSLParameters(); newParameters.setCipherSuites(validCiphers); newParameters.setProtocols(validProtocols); - newParameters.setNeedClientAuth(true); + switch (peerAuthentication) { + case WANT: + newParameters.setWantClientAuth(true); + break; + case NEED: + newParameters.setNeedClientAuth(true); + break; + case DISABLED: + break; + default: + throw new UnsupportedOperationException("Unknown peer authentication: " + peerAuthentication); + } return newParameters; } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/PeerAuthentication.java b/security-utils/src/main/java/com/yahoo/security/tls/PeerAuthentication.java new file mode 100644 index 00000000000..9aa7b642b4a --- /dev/null +++ b/security-utils/src/main/java/com/yahoo/security/tls/PeerAuthentication.java @@ -0,0 +1,9 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.security.tls; + +/** + * @author bjorncs + */ +public enum PeerAuthentication { + WANT, NEED, DISABLED +} diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java index b315dd00b31..ea26be0ef4f 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java @@ -4,6 +4,7 @@ package com.yahoo.security.tls; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLParameters; +import java.util.Set; /** * A simplified version of {@link SSLContext} modelled as an interface. @@ -12,6 +13,19 @@ import javax.net.ssl.SSLParameters; */ public interface TlsContext extends AutoCloseable { + Set<String> ALLOWED_CIPHER_SUITES = Set.of( + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_AES_128_GCM_SHA256", // TLSv1.3 + "TLS_AES_256_GCM_SHA384", // TLSv1.3 + "TLS_CHACHA20_POLY1305_SHA256"); // TLSv1.3 + + Set<String> ALLOWED_PROTOCOLS = Set.of("TLSv1.2"); // TODO Enable TLSv1.3 + SSLContext context(); SSLParameters parameters(); diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java b/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java index a4e508e0d2a..f28cad2a071 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java @@ -66,7 +66,7 @@ public class TransportSecurityUtils { public static Optional<TlsContext> createTlsContext() { return getConfigFile() - .map(configFile -> new ReloadingTlsContext(configFile, getInsecureAuthorizationMode())); + .map(configFile -> new ConfigFileBasedTlsContext(configFile, getInsecureAuthorizationMode())); } private static Optional<String> getEnvironmentVariable(Map<String, String> environmentVariables, String variableName) { diff --git a/security-utils/src/test/java/com/yahoo/security/tls/ReloadingTlsContextTest.java b/security-utils/src/test/java/com/yahoo/security/tls/ConfigFileBasedTlsContextTest.java index f991f86fdce..4e6f0a141b0 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/ReloadingTlsContextTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/ConfigFileBasedTlsContextTest.java @@ -26,7 +26,7 @@ import static org.assertj.core.api.Assertions.assertThat; /** * @author bjorncs */ -public class ReloadingTlsContextTest { +public class ConfigFileBasedTlsContextTest { @Rule public TemporaryFolder tempDirectory = new TemporaryFolder(); @@ -55,12 +55,12 @@ public class ReloadingTlsContextTest { Path optionsFile = tempDirectory.newFile().toPath(); options.toJsonFile(optionsFile); - try (TlsContext tlsContext = new ReloadingTlsContext(optionsFile, AuthorizationMode.ENFORCE)) { + try (TlsContext tlsContext = new ConfigFileBasedTlsContext(optionsFile, AuthorizationMode.ENFORCE)) { SSLEngine sslEngine = tlsContext.createSslEngine(); assertThat(sslEngine).isNotNull(); String[] enabledCiphers = sslEngine.getEnabledCipherSuites(); assertThat(enabledCiphers).isNotEmpty(); - assertThat(enabledCiphers).isSubsetOf(DefaultTlsContext.ALLOWED_CIPHER_SUITES.toArray(new String[0])); + assertThat(enabledCiphers).isSubsetOf(TlsContext.ALLOWED_CIPHER_SUITES.toArray(new String[0])); String[] enabledProtocols = sslEngine.getEnabledProtocols(); assertThat(enabledProtocols).contains("TLSv1.2"); diff --git a/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java b/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java index 5969d4d2ace..727a64ae934 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java @@ -15,7 +15,6 @@ import javax.security.auth.x500.X500Principal; import java.security.KeyPair; import java.security.cert.X509Certificate; import java.time.Instant; -import java.util.List; import static com.yahoo.security.KeyAlgorithm.EC; import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; @@ -47,13 +46,13 @@ public class DefaultTlsContextTest { singletonList(new RequiredPeerCredential(RequiredPeerCredential.Field.CN, new HostGlobPattern("dummy")))))); DefaultTlsContext tlsContext = - new DefaultTlsContext(singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers, AuthorizationMode.ENFORCE, List.of()); + new DefaultTlsContext(singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers, AuthorizationMode.ENFORCE, PeerAuthentication.NEED); SSLEngine sslEngine = tlsContext.createSslEngine(); assertThat(sslEngine).isNotNull(); String[] enabledCiphers = sslEngine.getEnabledCipherSuites(); assertThat(enabledCiphers).isNotEmpty(); - assertThat(enabledCiphers).isSubsetOf(DefaultTlsContext.ALLOWED_CIPHER_SUITES.toArray(new String[0])); + assertThat(enabledCiphers).isSubsetOf(TlsContext.ALLOWED_CIPHER_SUITES.toArray(new String[0])); String[] enabledProtocols = sslEngine.getEnabledProtocols(); assertThat(enabledProtocols).contains("TLSv1.2"); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java index 2b0e50ed982..cab28e55b21 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java @@ -3,23 +3,17 @@ package com.yahoo.vespa.athenz.identity; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.security.KeyStoreType; import com.yahoo.security.SslContextBuilder; +import com.yahoo.security.tls.AutoReloadingX509KeyManager; +import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.utils.SiaUtils; import javax.net.ssl.SSLContext; import java.io.File; import java.nio.file.Path; import java.nio.file.Paths; -import java.time.Duration; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; -import java.util.logging.Logger; /** * A {@link ServiceIdentityProvider} that provides the credentials stored on file system. @@ -29,24 +23,19 @@ import java.util.logging.Logger; */ public class SiaIdentityProvider extends AbstractComponent implements ServiceIdentityProvider { - private static final Logger log = Logger.getLogger(SiaIdentityProvider.class.getName()); - - private static final Duration REFRESH_INTERVAL = Duration.ofHours(1); - - private final AtomicReference<SSLContext> sslContext = new AtomicReference<>(); + private final AutoReloadingX509KeyManager keyManager; + private final SSLContext sslContext; private final AthenzIdentity service; private final File privateKeyFile; private final File certificateFile; private final File trustStoreFile; - private final ScheduledExecutorService scheduler; @Inject public SiaIdentityProvider(SiaProviderConfig config) { this(new AthenzService(config.athenzDomain(), config.athenzService()), SiaUtils.getPrivateKeyFile(Paths.get(config.keyPathPrefix()), new AthenzService(config.athenzDomain(), config.athenzService())).toFile(), SiaUtils.getCertificateFile(Paths.get(config.keyPathPrefix()), new AthenzService(config.athenzDomain(), config.athenzService())).toFile(), - new File(config.trustStorePath()), - createScheduler()); + new File(config.trustStorePath())); } public SiaIdentityProvider(AthenzIdentity service, @@ -55,30 +44,19 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde this(service, SiaUtils.getPrivateKeyFile(siaPath, service).toFile(), SiaUtils.getCertificateFile(siaPath, service).toFile(), - trustStoreFile, - createScheduler()); + trustStoreFile); } public SiaIdentityProvider(AthenzIdentity service, File privateKeyFile, File certificateFile, - File trustStoreFile, - ScheduledExecutorService scheduler) { + File trustStoreFile) { this.service = service; this.privateKeyFile = privateKeyFile; this.certificateFile = certificateFile; this.trustStoreFile = trustStoreFile; - this.scheduler = scheduler; - this.sslContext.set(createIdentitySslContext()); - scheduler.scheduleAtFixedRate(this::reloadSslContext, REFRESH_INTERVAL.toMinutes(), REFRESH_INTERVAL.toMinutes(), TimeUnit.MINUTES); - } - - private static ScheduledThreadPoolExecutor createScheduler() { - return new ScheduledThreadPoolExecutor(1, runnable -> { - Thread thread = new Thread(runnable); - thread.setName("sia-identity-provider-sslcontext-updater"); - return thread; - }); + this.keyManager = AutoReloadingX509KeyManager.fromPemFiles(privateKeyFile.toPath(), certificateFile.toPath()); + this.sslContext = createIdentitySslContext(keyManager, trustStoreFile.toPath()); } @Override @@ -88,34 +66,18 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde @Override public SSLContext getIdentitySslContext() { - return sslContext.get(); + return sslContext; } - private SSLContext createIdentitySslContext() { + private static SSLContext createIdentitySslContext(AutoReloadingX509KeyManager keyManager, Path trustStoreFile) { return new SslContextBuilder() - .withTrustStore(trustStoreFile.toPath(), KeyStoreType.JKS) - .withKeyStore(privateKeyFile.toPath(), certificateFile.toPath()) + .withTrustStore(trustStoreFile, KeyStoreType.JKS) + .withKeyManager(keyManager) .build(); } - private void reloadSslContext() { - log.log(LogLevel.DEBUG, "Updating SSLContext for identity " + service.getFullName()); - try { - SSLContext sslContext = createIdentitySslContext(); - this.sslContext.set(sslContext); - } catch (Exception e) { - log.log(LogLevel.SEVERE, "Failed to update SSLContext: " + e.getMessage(), e); - } - } - - @Override public void deconstruct() { - try { - scheduler.shutdownNow(); - scheduler.awaitTermination(90, TimeUnit.SECONDS); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + keyManager.close(); } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentials.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentials.java index a1d8a9ca258..d4494c1bd26 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentials.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentials.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.athenz.identityprovider.client; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import javax.net.ssl.SSLContext; import java.security.KeyPair; import java.security.cert.X509Certificate; @@ -15,16 +14,13 @@ class AthenzCredentials { private final X509Certificate certificate; private final KeyPair keyPair; private final SignedIdentityDocument identityDocument; - private final SSLContext identitySslContext; AthenzCredentials(X509Certificate certificate, KeyPair keyPair, - SignedIdentityDocument identityDocument, - SSLContext identitySslContext) { + SignedIdentityDocument identityDocument) { this.certificate = certificate; this.keyPair = keyPair; this.identityDocument = identityDocument; - this.identitySslContext = identitySslContext; } X509Certificate getCertificate() { @@ -39,7 +35,4 @@ class AthenzCredentials { return identityDocument; } - SSLContext getIdentitySslContext() { - return identitySslContext; - } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java index 39d0db4affd..9e2d8bc548c 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.athenz.identityprovider.client; import com.yahoo.container.core.identity.IdentityConfig; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; -import com.yahoo.security.SslContextBuilder; +import com.yahoo.security.Pkcs10Csr; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; import com.yahoo.vespa.athenz.client.zts.InstanceIdentity; @@ -14,12 +14,10 @@ import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocumentClient; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; -import com.yahoo.security.Pkcs10Csr; import com.yahoo.vespa.athenz.utils.SiaUtils; import com.yahoo.vespa.defaults.Defaults; import javax.net.ssl.SSLContext; -import java.io.File; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; @@ -31,7 +29,6 @@ import java.time.Clock; import java.time.Duration; import java.util.Optional; -import static com.yahoo.security.KeyStoreType.JKS; import static java.util.Collections.singleton; /** @@ -49,14 +46,12 @@ class AthenzCredentialsService { private final URI ztsEndpoint; private final AthenzService configserverIdentity; private final ServiceIdentityProvider nodeIdentityProvider; - private final File trustStoreJks; private final String hostname; private final CsrGenerator csrGenerator; private final Clock clock; AthenzCredentialsService(IdentityConfig identityConfig, ServiceIdentityProvider nodeIdentityProvider, - File trustStoreJks, String hostname, Clock clock) { this.tenantIdentity = new AthenzService(identityConfig.domain(), identityConfig.service()); @@ -64,7 +59,6 @@ class AthenzCredentialsService { this.ztsEndpoint = URI.create(identityConfig.ztsUrl()); this.configserverIdentity = new AthenzService(identityConfig.configserverIdentityName()); this.nodeIdentityProvider = nodeIdentityProvider; - this.trustStoreJks = trustStoreJks; this.hostname = hostname; this.csrGenerator = new CsrGenerator(identityConfig.athenzDnsSuffix(), identityConfig.configserverIdentityName()); this.clock = clock; @@ -94,9 +88,8 @@ class AthenzCredentialsService { false, csr); X509Certificate certificate = instanceIdentity.certificate(); - SSLContext identitySslContext = createIdentitySslContext(keyPair.getPrivate(), certificate); writeCredentialsToDisk(keyPair.getPrivate(), certificate, document); - return new AthenzCredentials(certificate, keyPair, document, identitySslContext); + return new AthenzCredentials(certificate, keyPair, document); } } @@ -117,9 +110,8 @@ class AthenzCredentialsService { false, csr); X509Certificate certificate = instanceIdentity.certificate(); - SSLContext identitySslContext = createIdentitySslContext(newKeyPair.getPrivate(), certificate); writeCredentialsToDisk(newKeyPair.getPrivate(), certificate, document); - return new AthenzCredentials(certificate, newKeyPair, document, identitySslContext); + return new AthenzCredentials(certificate, newKeyPair, document); } } @@ -134,8 +126,7 @@ class AthenzCredentialsService { if (Files.notExists(IDENTITY_DOCUMENT_FILE)) return Optional.empty(); SignedIdentityDocument signedIdentityDocument = EntityBindingsMapper.readSignedIdentityDocumentFromFile(IDENTITY_DOCUMENT_FILE); KeyPair keyPair = new KeyPair(KeyUtils.extractPublicKey(privateKey.get()), privateKey.get()); - SSLContext sslContext = createIdentitySslContext(privateKey.get(), certificate.get()); - return Optional.of(new AthenzCredentials(certificate.get(), keyPair, signedIdentityDocument, sslContext)); + return Optional.of(new AthenzCredentials(certificate.get(), keyPair, signedIdentityDocument)); } private boolean isExpired(X509Certificate certificate) { @@ -150,13 +141,6 @@ class AthenzCredentialsService { EntityBindingsMapper.writeSignedIdentityDocumentToFile(IDENTITY_DOCUMENT_FILE, identityDocument); } - private SSLContext createIdentitySslContext(PrivateKey privateKey, X509Certificate certificate) { - return new SslContextBuilder() - .withKeyStore(privateKey, certificate) - .withTrustStore(trustStoreJks.toPath(), JKS) - .build(); - } - private DefaultIdentityDocumentClient createIdentityDocumentClient() { return new DefaultIdentityDocumentClient( configserverEndpoint, diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java index ac255289883..e4633fb708b 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java @@ -12,8 +12,11 @@ import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider; import com.yahoo.container.jdisc.athenz.AthenzIdentityProviderException; import com.yahoo.jdisc.Metric; import com.yahoo.log.LogLevel; +import com.yahoo.security.KeyStoreBuilder; import com.yahoo.security.KeyStoreType; +import com.yahoo.security.Pkcs10Csr; import com.yahoo.security.SslContextBuilder; +import com.yahoo.security.tls.MutableX509KeyManager; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.AthenzService; @@ -22,13 +25,14 @@ import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; import com.yahoo.vespa.athenz.client.zts.ZtsClient; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.identity.SiaIdentityProvider; -import com.yahoo.security.Pkcs10Csr; import com.yahoo.vespa.athenz.utils.SiaUtils; import com.yahoo.vespa.defaults.Defaults; import javax.net.ssl.SSLContext; -import java.io.File; +import javax.net.ssl.X509ExtendedKeyManager; import java.net.URI; +import java.nio.file.Path; +import java.nio.file.Paths; import java.security.PrivateKey; import java.security.cert.X509Certificate; import java.time.Clock; @@ -42,6 +46,9 @@ import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.logging.Logger; +import static com.yahoo.security.KeyStoreType.JKS; +import static com.yahoo.security.KeyStoreType.PKCS12; + /** * A {@link AthenzIdentityProvider} / {@link ServiceIdentityProvider} component that provides the tenant identity. * @@ -59,10 +66,14 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen private final static Duration ROLE_SSL_CONTEXT_EXPIRY = Duration.ofHours(24); private final static Duration ROLE_TOKEN_EXPIRY = Duration.ofMinutes(30); + // TODO Make path to trust store config + private static final Path DEFAULT_TRUST_STORE = Paths.get(Defaults.getDefaults().underVespaHome("share/ssl/certs/yahoo_certificate_bundle.pem")); + public static final String CERTIFICATE_EXPIRY_METRIC_NAME = "athenz-tenant-cert.expiry.seconds"; private volatile AthenzCredentials credentials; private final Metric metric; + private final Path trustStore; private final AthenzCredentialsService athenzCredentialsService; private final ScheduledExecutorService scheduler; private final Clock clock; @@ -70,6 +81,8 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen private final String dnsSuffix; private final URI ztsEndpoint; + private final MutableX509KeyManager identityKeyManager = new MutableX509KeyManager(); + private final SSLContext identitySslContext; private final LoadingCache<AthenzRole, SSLContext> roleSslContextCache; private final LoadingCache<AthenzRole, ZToken> roleSpecificRoleTokenCache; private final LoadingCache<AthenzDomain, ZToken> domainSpecificRoleTokenCache; @@ -79,9 +92,9 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen public AthenzIdentityProviderImpl(IdentityConfig config, Metric metric) { this(config, metric, + DEFAULT_TRUST_STORE, new AthenzCredentialsService(config, - createNodeIdentityProvider(config), - getDefaultTrustStoreLocation(), + createNodeIdentityProvider(config, DEFAULT_TRUST_STORE), Defaults.getDefaults().vespaHostname(), Clock.systemUTC()), new ScheduledThreadPoolExecutor(1), @@ -92,10 +105,12 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen AthenzIdentityProviderImpl(IdentityConfig config, Metric metric, + Path trustStore, AthenzCredentialsService athenzCredentialsService, ScheduledExecutorService scheduler, Clock clock) { this.metric = metric; + this.trustStore = trustStore; this.athenzCredentialsService = athenzCredentialsService; this.scheduler = scheduler; this.clock = clock; @@ -106,6 +121,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen roleSpecificRoleTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createRoleToken); domainSpecificRoleTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createRoleToken); this.csrGenerator = new CsrGenerator(config.athenzDnsSuffix(), config.configserverIdentityName()); + this.identitySslContext = createIdentitySslContext(identityKeyManager, trustStore); registerInstance(); } @@ -121,11 +137,18 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen }); } + private static SSLContext createIdentitySslContext(X509ExtendedKeyManager keyManager, Path trustStore) { + return new SslContextBuilder() + .withKeyManager(keyManager) + .withTrustStore(trustStore, JKS) + .build(); + } + private void registerInstance() { try { - credentials = athenzCredentialsService.registerInstance(); - scheduler.scheduleAtFixedRate(this::refreshCertificate, UPDATE_PERIOD.toMinutes(), UPDATE_PERIOD.toMinutes(), TimeUnit.MINUTES); - scheduler.scheduleAtFixedRate(this::reportMetrics, 0, 5, TimeUnit.MINUTES); + updateIdentityCredentials(this.athenzCredentialsService.registerInstance()); + this.scheduler.scheduleAtFixedRate(this::refreshCertificate, UPDATE_PERIOD.toMinutes(), UPDATE_PERIOD.toMinutes(), TimeUnit.MINUTES); + this.scheduler.scheduleAtFixedRate(this::reportMetrics, 0, 5, TimeUnit.MINUTES); } catch (Throwable t) { throw new AthenzIdentityProviderException("Could not retrieve Athenz credentials", t); } @@ -148,7 +171,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen @Override public SSLContext getIdentitySslContext() { - return credentials.getIdentitySslContext(); + return identitySslContext; } @Override @@ -189,13 +212,22 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen return Collections.singletonList(credentials.getCertificate()); } + private void updateIdentityCredentials(AthenzCredentials credentials) { + this.credentials = credentials; + this.identityKeyManager.updateKeystore( + KeyStoreBuilder.withType(PKCS12) + .withKeyEntry("default", credentials.getKeyPair().getPrivate(), credentials.getCertificate()) + .build(), + new char[0]); + } + private SSLContext createRoleSslContext(AthenzRole role) { Pkcs10Csr csr = csrGenerator.generateRoleCsr(identity, role, credentials.getIdentityDocument().providerUniqueId(), credentials.getKeyPair()); try (ZtsClient client = createZtsClient()) { X509Certificate roleCertificate = client.getRoleCertificate(role, csr); return new SslContextBuilder() .withKeyStore(credentials.getKeyPair().getPrivate(), roleCertificate) - .withTrustStore(getDefaultTrustStoreLocation().toPath(), KeyStoreType.JKS) + .withTrustStore(trustStore, KeyStoreType.JKS) .build(); } } @@ -226,13 +258,9 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen } } - private static SiaIdentityProvider createNodeIdentityProvider(IdentityConfig config) { + private static SiaIdentityProvider createNodeIdentityProvider(IdentityConfig config, Path trustStore) { return new SiaIdentityProvider( - new AthenzService(config.nodeIdentityName()), SiaUtils.DEFAULT_SIA_DIRECTORY, getDefaultTrustStoreLocation()); - } - - private static File getDefaultTrustStoreLocation() { - return new File(Defaults.getDefaults().underVespaHome("share/ssl/certs/yahoo_certificate_bundle.jks")); + new AthenzService(config.nodeIdentityName()), SiaUtils.DEFAULT_SIA_DIRECTORY, trustStore.toFile()); } private boolean isExpired(AthenzCredentials credentials) { @@ -245,9 +273,9 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen void refreshCertificate() { try { - credentials = isExpired(credentials) - ? athenzCredentialsService.registerInstance() - : athenzCredentialsService.updateCredentials(credentials.getIdentityDocument(), credentials.getIdentitySslContext()); + updateIdentityCredentials(isExpired(credentials) + ? athenzCredentialsService.registerInstance() + : athenzCredentialsService.updateCredentials(credentials.getIdentityDocument(), identitySslContext)); } catch (Throwable t) { log.log(LogLevel.WARNING, "Failed to update credentials: " + t.getMessage(), t); } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identity/SiaIdentityProviderTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identity/SiaIdentityProviderTest.java index 0195d6000e1..31152a4602f 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identity/SiaIdentityProviderTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identity/SiaIdentityProviderTest.java @@ -24,10 +24,8 @@ import java.security.KeyStore; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; -import java.util.concurrent.ScheduledExecutorService; import static org.junit.Assert.assertNotNull; -import static org.mockito.Mockito.mock; /** * @author bjorncs @@ -55,8 +53,7 @@ public class SiaIdentityProviderTest { new AthenzService("domain", "service-name"), keyFile, certificateFile, - trustStoreFile, - mock(ScheduledExecutorService.class)); + trustStoreFile); assertNotNull(provider.getIdentitySslContext()); } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java index 01dab2dada3..c584b803815 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java @@ -4,14 +4,30 @@ package com.yahoo.vespa.athenz.identityprovider.client; import com.yahoo.container.core.identity.IdentityConfig; import com.yahoo.container.jdisc.athenz.AthenzIdentityProviderException; import com.yahoo.jdisc.Metric; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyStoreBuilder; +import com.yahoo.security.KeyStoreType; +import com.yahoo.security.KeyStoreUtils; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.Pkcs10Csr; +import com.yahoo.security.Pkcs10CsrBuilder; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; import com.yahoo.test.ManualClock; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import javax.security.auth.x500.X500Principal; +import java.io.IOException; +import java.math.BigInteger; +import java.nio.file.Path; +import java.security.KeyPair; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Date; import java.util.concurrent.ScheduledExecutorService; import java.util.function.Supplier; @@ -43,13 +59,36 @@ public class AthenzIdentityProviderImplTest { .ztsUrl("https:localhost:4443/zts/v1") .athenzDnsSuffix("dev-us-north-1.vespa.cloud")); + private final KeyPair caKeypair = KeyUtils.generateKeypair(KeyAlgorithm.EC); + private Path trustStoreFile; + private X509Certificate caCertificate; + + @Before + public void createTrustStoreFile() throws IOException { + caCertificate = X509CertificateBuilder + .fromKeypair( + caKeypair, + new X500Principal("CN=mydummyca"), + Instant.EPOCH, + Instant.EPOCH.plus(10000, ChronoUnit.DAYS), + SignatureAlgorithm.SHA256_WITH_ECDSA, + BigInteger.ONE) + .build(); + trustStoreFile = tempDir.newFile().toPath(); + KeyStoreUtils.writeKeyStoreToFile( + KeyStoreBuilder.withType(KeyStoreType.JKS) + .withKeyEntry("default", caKeypair.getPrivate(), caCertificate) + .build(), + trustStoreFile); + } + @Test(expected = AthenzIdentityProviderException.class) public void component_creation_fails_when_credentials_not_found() { AthenzCredentialsService credentialService = mock(AthenzCredentialsService.class); when(credentialService.registerInstance()) .thenThrow(new RuntimeException("athenz unavailable")); - new AthenzIdentityProviderImpl(IDENTITY_CONFIG, mock(Metric.class), credentialService, mock(ScheduledExecutorService.class), new ManualClock(Instant.EPOCH)); + new AthenzIdentityProviderImpl(IDENTITY_CONFIG, mock(Metric.class), trustStoreFile ,credentialService, mock(ScheduledExecutorService.class), new ManualClock(Instant.EPOCH)); } @Test @@ -59,18 +98,19 @@ public class AthenzIdentityProviderImplTest { AthenzCredentialsService athenzCredentialsService = mock(AthenzCredentialsService.class); - X509Certificate certificate = getCertificate(getExpirationSupplier(clock)); + KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC); + X509Certificate certificate = getCertificate(keyPair, getExpirationSupplier(clock)); when(athenzCredentialsService.registerInstance()) - .thenReturn(new AthenzCredentials(certificate, null, null, null)); + .thenReturn(new AthenzCredentials(certificate, keyPair, null)); when(athenzCredentialsService.updateCredentials(any(), any())) .thenThrow(new RuntimeException("#1")) .thenThrow(new RuntimeException("#2")) - .thenReturn(new AthenzCredentials(certificate, null, null, null)); + .thenReturn(new AthenzCredentials(certificate, keyPair, null)); AthenzIdentityProviderImpl identityProvider = - new AthenzIdentityProviderImpl(IDENTITY_CONFIG, metric, athenzCredentialsService, mock(ScheduledExecutorService.class), clock); + new AthenzIdentityProviderImpl(IDENTITY_CONFIG, metric, trustStoreFile, athenzCredentialsService, mock(ScheduledExecutorService.class), clock); identityProvider.reportMetrics(); verify(metric).set(eq(AthenzIdentityProviderImpl.CERTIFICATE_EXPIRY_METRIC_NAME), eq(certificateValidity.getSeconds()), any()); @@ -99,10 +139,18 @@ public class AthenzIdentityProviderImplTest { return () -> new Date(clock.instant().plus(certificateValidity).toEpochMilli()); } - private X509Certificate getCertificate(Supplier<Date> expiry) { - X509Certificate x509Certificate = mock(X509Certificate.class); - when(x509Certificate.getNotAfter()).thenReturn(expiry.get()); - return x509Certificate; + private X509Certificate getCertificate(KeyPair keyPair, Supplier<Date> expiry) { + Pkcs10Csr csr = Pkcs10CsrBuilder.fromKeypair(new X500Principal("CN=dummy"), keyPair, SignatureAlgorithm.SHA256_WITH_ECDSA) + .build(); + return X509CertificateBuilder + .fromCsr(csr, + caCertificate.getSubjectX500Principal(), + Instant.EPOCH, + expiry.get().toInstant(), + caKeypair.getPrivate(), + SignatureAlgorithm.SHA256_WITH_ECDSA, + BigInteger.ONE) + .build(); } } diff --git a/vespa-hadoop/pom.xml b/vespa-hadoop/pom.xml index 999bb7bcc01..1ee29c36b7d 100644 --- a/vespa-hadoop/pom.xml +++ b/vespa-hadoop/pom.xml @@ -126,16 +126,46 @@ </goals> <configuration> <minimizeJar>false</minimizeJar> + <relocations> <relocation> - <pattern>org.apache.http</pattern> - <shadedPattern>com.yahoo.vespa.feeder.shaded.internal.apache.http</shadedPattern> + <pattern>com.google</pattern> + <shadedPattern>shaded.vespa</shadedPattern> + </relocation> + <relocation> + <pattern>commons-codec</pattern> + <shadedPattern>shaded.vespa</shadedPattern> + </relocation> + <relocation> + <pattern>commons-logging</pattern> + <shadedPattern>shaded.vespa</shadedPattern> + </relocation> + <relocation> + <pattern>org.apache</pattern> + <shadedPattern>shaded.vespa</shadedPattern> + <excludes> + <exclude>org.apache.hadoop.**</exclude> + <exclude>org.apache.pig.**</exclude> + </excludes> </relocation> <relocation> - <pattern>org.apache.commons</pattern> - <shadedPattern>com.yahoo.vespa.feeder.shaded.internal.apache.commons</shadedPattern> + <pattern>com.fasterxml</pattern> + <shadedPattern>shaded.vespa</shadedPattern> + </relocation> + <relocation> + <pattern>org.codehaus</pattern> + <shadedPattern>shaded.vespa</shadedPattern> + </relocation> + <relocation> + <pattern>io.airlift</pattern> + <shadedPattern>shaded.vespa</shadedPattern> + </relocation> + <relocation> + <pattern>com.ctc.wstx</pattern> + <shadedPattern>shaded.vespa</shadedPattern> </relocation> </relocations> + </configuration> </execution> </executions> |