summaryrefslogtreecommitdiffstats
path: root/controller-server
diff options
context:
space:
mode:
authorØyvind Grønnesby <oyving@yahooinc.com>2022-05-30 12:28:14 +0200
committerØyvind Grønnesby <oyving@yahooinc.com>2022-05-30 12:28:14 +0200
commit9fa7903909043d7b855f7e3ba315050ba5a12597 (patch)
tree3b6654d0725d64ac27f92ae089f0897895b7059a /controller-server
parent45d46ce6c3b3d8c85f1ec9cfbd3793a161571265 (diff)
parente657c0a9618868c9dcf32cfa7e05ac73750b904c (diff)
Merge remote-tracking branch 'origin/master' into ogronnesby/contact-info-resources
Diffstat (limited to 'controller-server')
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java13
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java28
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java35
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java4
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java11
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java3
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java25
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java13
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java4
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java4
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java20
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java13
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java11
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializerTest.java14
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java9
16 files changed, 136 insertions, 73 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index 6907747646e..e48ad7596ea 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -7,6 +7,7 @@ import com.yahoo.config.application.api.DeploymentSpec;
import com.yahoo.config.application.api.ValidationId;
import com.yahoo.config.application.api.ValidationOverrides;
import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.CloudAccount;
import com.yahoo.config.provision.DockerImage;
import com.yahoo.config.provision.InstanceName;
import com.yahoo.config.provision.TenantName;
@@ -19,7 +20,6 @@ import com.yahoo.vespa.athenz.api.AthenzIdentity;
import com.yahoo.vespa.athenz.api.AthenzPrincipal;
import com.yahoo.vespa.athenz.api.AthenzService;
import com.yahoo.vespa.athenz.api.AthenzUser;
-import com.yahoo.vespa.curator.Lock;
import com.yahoo.vespa.flags.FetchVector;
import com.yahoo.vespa.flags.FlagSource;
import com.yahoo.vespa.flags.ListFlag;
@@ -41,7 +41,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationS
import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId;
import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter;
@@ -60,9 +59,7 @@ import com.yahoo.vespa.hosted.controller.certificate.EndpointCertificates;
import com.yahoo.vespa.hosted.controller.concurrent.Once;
import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger;
import com.yahoo.vespa.hosted.controller.deployment.JobStatus;
-import com.yahoo.vespa.hosted.controller.deployment.RevisionHistory;
import com.yahoo.vespa.hosted.controller.deployment.Run;
-import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
import com.yahoo.vespa.hosted.controller.notification.Notification;
import com.yahoo.vespa.hosted.controller.notification.NotificationSource;
import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
@@ -85,7 +82,6 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -93,7 +89,6 @@ import java.util.Optional;
import java.util.OptionalInt;
import java.util.Set;
import java.util.TreeMap;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.function.Predicate;
@@ -620,12 +615,14 @@ public class ApplicationController {
List<X509Certificate> operatorCertificates = controller.supportAccess().activeGrantsFor(deployment).stream()
.map(SupportAccessGrant::certificate)
.collect(toList());
-
+ Optional<CloudAccount> cloudAccount = applicationPackage.deploymentSpec()
+ .instance(application.instance())
+ .flatMap(spec -> spec.cloudAccount(zone.environment(), zone.region()));
ConfigServer.PreparedApplication preparedApplication =
configServer.deploy(new DeploymentData(application, zone, applicationPackage.zippedContent(), platform,
endpoints, endpointCertificateMetadata, dockerImageRepo, domain,
deploymentQuota, tenantSecretStores, operatorCertificates,
- dryRun));
+ cloudAccount, dryRun));
return new ActivateResult(new com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(),
applicationPackage.zippedContent().length);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java
index 950eaea904a..ccad4fe92ad 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java
@@ -6,11 +6,16 @@ import com.yahoo.config.application.api.DeploymentSpec;
import com.yahoo.config.application.api.Endpoint;
import com.yahoo.config.application.api.ValidationId;
import com.yahoo.config.application.api.ValidationOverrides;
+import com.yahoo.config.provision.CloudAccount;
import com.yahoo.config.provision.CloudName;
import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.TenantName;
import com.yahoo.config.provision.zone.ZoneApi;
import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.ListFlag;
+import com.yahoo.vespa.flags.PermanentFlags;
import com.yahoo.vespa.hosted.controller.Application;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.application.EndpointId;
@@ -24,6 +29,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
+import java.util.Set;
import java.util.stream.Collectors;
/**
@@ -34,9 +40,11 @@ import java.util.stream.Collectors;
public class ApplicationPackageValidator {
private final Controller controller;
+ private final ListFlag<String> cloudAccountsFlag;
public ApplicationPackageValidator(Controller controller) {
this.controller = Objects.requireNonNull(controller, "controller must be non-null");
+ this.cloudAccountsFlag = PermanentFlags.CLOUD_ACCOUNTS.bindTo(controller.flagSource());
}
/**
@@ -46,6 +54,7 @@ public class ApplicationPackageValidator {
*/
public void validate(Application application, ApplicationPackage applicationPackage, Instant instant) {
validateSteps(applicationPackage.deploymentSpec());
+ validateCloudAccounts(application, applicationPackage.deploymentSpec());
validateEndpointRegions(applicationPackage.deploymentSpec());
validateEndpointChange(application, applicationPackage, instant);
validateCompactedEndpoint(applicationPackage);
@@ -145,6 +154,25 @@ public class ApplicationPackageValidator {
". " + ValidationOverrides.toAllowMessage(validationId));
}
+ /** Verify that declared cloud accounts are allowed to be used by the tenant */
+ private void validateCloudAccounts(Application application, DeploymentSpec deploymentSpec) {
+ TenantName tenant = application.id().tenant();
+ Set<CloudAccount> validAccounts = cloudAccountsFlag.with(FetchVector.Dimension.TENANT_ID, tenant.value())
+ .value().stream()
+ .map(CloudAccount::new)
+ .collect(Collectors.toSet());
+ for (var spec : deploymentSpec.instances()) {
+ for (var zone : spec.zones()) {
+ if (!zone.environment().isProduction()) continue;
+ Optional<CloudAccount> cloudAccount = spec.cloudAccount(zone.environment(), zone.region().get());
+ if (cloudAccount.isEmpty()) continue;
+ if (validAccounts.contains(cloudAccount.get())) continue;
+ throw new IllegalArgumentException("Cloud account '" + cloudAccount.get().value() +
+ "' is not valid for tenant '" + tenant + "'");
+ }
+ }
+ }
+
/** Returns whether newEndpoints contains all destinations in endpoints */
private static boolean containsAllDestinationsOf(List<Endpoint> endpoints, List<Endpoint> newEndpoints) {
var containsAllRegions = true;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java
index 19fb1b7e1bb..6d011438b10 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java
@@ -1,7 +1,6 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.auditlog;
-import com.google.common.base.CharMatcher;
import com.google.common.collect.Ordering;
import java.time.Instant;
@@ -10,7 +9,6 @@ import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
-import java.util.function.Predicate;
/**
* This represents the audit log of a hosted Vespa system. The audit log contains manual actions performed through
@@ -66,7 +64,7 @@ public class AuditLog {
private final String resource;
private final Optional<String> data;
- public Entry(Instant at, String principal, Method method, String resource, Optional<String> data) {
+ public Entry(Instant at, String principal, Method method, String resource, byte[] data) {
this.at = Objects.requireNonNull(at, "at must be non-null");
this.principal = Objects.requireNonNull(principal, "principal must be non-null");
this.method = Objects.requireNonNull(method, "method must be non-null");
@@ -112,16 +110,27 @@ public class AuditLog {
DELETE
}
- private static Optional<String> sanitize(Optional<String> data) {
- Objects.requireNonNull(data, "data must be non-null");
- return data.filter(Predicate.not(String::isBlank))
- .filter(CharMatcher.ascii()::matchesAllOf)
- .map(v -> {
- if (v.length() > maxDataLength) {
- return v.substring(0, maxDataLength);
- }
- return v;
- });
+ private static Optional<String> sanitize(byte[] data) {
+ StringBuilder sb = new StringBuilder();
+ for (byte b : data) {
+ char c = (char) b;
+ if (!printableAscii(c) && !tabOrLineBreak(c)) {
+ return Optional.empty();
+ }
+ sb.append(c);
+ if (sb.length() == maxDataLength) {
+ break;
+ }
+ }
+ return Optional.of(sb.toString()).filter(s -> !s.isEmpty());
+ }
+
+ private static boolean printableAscii(char c) {
+ return c >= 32 && c <= 126;
+ }
+
+ private static boolean tabOrLineBreak(char c) {
+ return c == 9 || c == 10 || c == 13;
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java
index 34e7955e02a..b6782767386 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java
@@ -3,14 +3,12 @@ package com.yahoo.vespa.hosted.controller.auditlog;
import com.yahoo.container.jdisc.HttpRequest;
import com.yahoo.transaction.Mutex;
-import com.yahoo.vespa.curator.Lock;
import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.URI;
-import java.nio.charset.StandardCharsets;
import java.security.Principal;
import java.time.Clock;
import java.time.Duration;
@@ -70,7 +68,7 @@ public class AuditLogger {
Instant now = clock.instant();
AuditLog.Entry entry = new AuditLog.Entry(now, principal.getName(), method.get(), pathAndQueryOf(request.getUri()),
- Optional.of(new String(data, StandardCharsets.UTF_8)));
+ data);
try (Mutex lock = db.lockAuditLog()) {
AuditLog auditLog = db.readAuditLog()
.pruneBefore(now.minus(entryTtl))
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 52e5431b552..fb7a3fc4ba5 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -13,11 +13,6 @@ import com.yahoo.config.provision.SystemName;
import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.log.LogLevel;
-import com.yahoo.security.KeyAlgorithm;
-import com.yahoo.security.KeyUtils;
-import com.yahoo.security.SignatureAlgorithm;
-import com.yahoo.security.X509CertificateBuilder;
-import com.yahoo.security.X509CertificateUtils;
import com.yahoo.vespa.hosted.controller.Application;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.Instance;
@@ -48,12 +43,9 @@ import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext;
import com.yahoo.yolean.Exceptions;
-import javax.security.auth.x500.X500Principal;
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
import java.io.UncheckedIOException;
-import java.math.BigInteger;
-import java.security.KeyPair;
import java.security.cert.CertificateExpiredException;
import java.security.cert.CertificateNotYetValidException;
import java.security.cert.X509Certificate;
@@ -65,7 +57,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
-import java.util.UUID;
import java.util.function.Consumer;
import java.util.function.Supplier;
import java.util.logging.Level;
@@ -95,7 +86,6 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal;
import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester;
import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester;
import static com.yahoo.vespa.hosted.controller.deployment.Step.report;
-import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Objects.requireNonNull;
import static java.util.logging.Level.FINE;
import static java.util.logging.Level.INFO;
@@ -671,6 +661,7 @@ public class InternalStepRunner implements StepRunner {
logger.log(INFO, "The test package should either contain basic HTTP tests under 'tests/<suite-name>/', " +
"or a Java test bundle under 'components/' with at least one test with the annotation " +
"for this suite. See docs.vespa.ai/en/testing.html for details.");
+ controller.jobController().updateTestReport(id);
return Optional.of(noTests);
}
case SUCCESS:
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java
index 81fbb85e3d7..439f1aa9a09 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java
@@ -53,7 +53,7 @@ public class ArtifactExpirer extends ControllerMaintainer {
.filter(artifact -> isExpired(artifact, now, versionStatus))
.collect(Collectors.toList());
if (!artifactsToExpire.isEmpty()) {
- log.log(Level.INFO, "Expiring " + artifactsToExpire.size() + " artifacts: " + artifactsToExpire);
+ log.log(Level.INFO, "Expiring " + artifactsToExpire.size() + " artifacts in " + cloudName + ": " + artifactsToExpire);
artifactRegistry.deleteAll(artifactsToExpire);
}
return 1;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java
index c94c04fc244..38b9d994a6d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java
@@ -8,6 +8,7 @@ import com.yahoo.slime.Slime;
import com.yahoo.slime.SlimeUtils;
import com.yahoo.vespa.hosted.controller.auditlog.AuditLog;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
@@ -57,6 +58,8 @@ public class AuditLogSerializer {
methodFrom(entryObject.field(methodField)),
entryObject.field(resourceField).asString(),
SlimeUtils.optionalString(entryObject.field(dataField))
+ .map(s -> s.getBytes(StandardCharsets.UTF_8))
+ .orElseGet(() -> new byte[0])
));
});
return new AuditLog(entries);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java
index 671222e2123..9bea7fb829d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java
@@ -1,13 +1,13 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.proxy;
-import com.yahoo.component.annotation.Inject;
import com.yahoo.component.AbstractComponent;
+import com.yahoo.component.annotation.Inject;
import com.yahoo.jdisc.http.HttpRequest.Method;
import com.yahoo.text.Text;
import com.yahoo.vespa.athenz.api.AthenzIdentity;
-import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier;
+import com.yahoo.vespa.hosted.controller.api.integration.ControllerIdentityProvider;
import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
import com.yahoo.yolean.concurrent.Sleeper;
import org.apache.http.Header;
@@ -20,6 +20,7 @@ import org.apache.http.client.methods.HttpPatch;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.entity.InputStreamEntity;
import org.apache.http.impl.DefaultConnectionReuseStrategy;
import org.apache.http.impl.client.CloseableHttpClient;
@@ -29,7 +30,6 @@ import org.apache.http.protocol.HttpCoreContext;
import org.apache.http.util.EntityUtils;
import javax.net.ssl.HostnameVerifier;
-import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSession;
import java.io.IOException;
import java.io.InputStream;
@@ -68,16 +68,15 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C
private final Sleeper sleeper;
@Inject
- public ConfigServerRestExecutorImpl(ZoneRegistry zoneRegistry, ServiceIdentityProvider sslContextProvider) {
- this(zoneRegistry, sslContextProvider.getIdentitySslContext(), Sleeper.DEFAULT,
- new ConnectionReuseStrategy(zoneRegistry));
+ public ConfigServerRestExecutorImpl(ZoneRegistry zoneRegistry, ControllerIdentityProvider identityProvider) {
+ this(new SSLConnectionSocketFactory(identityProvider.getConfigServerSslSocketFactory(), new ControllerOrConfigserverHostnameVerifier(zoneRegistry)),
+ Sleeper.DEFAULT,
+ new ConnectionReuseStrategy(zoneRegistry));
}
- ConfigServerRestExecutorImpl(ZoneRegistry zoneRegistry, SSLContext sslContext,
+ ConfigServerRestExecutorImpl(SSLConnectionSocketFactory connectionSocketFactory,
Sleeper sleeper, ConnectionReuseStrategy connectionReuseStrategy) {
- this.client = createHttpClient(sslContext,
- new ControllerOrConfigserverHostnameVerifier(zoneRegistry),
- connectionReuseStrategy);
+ this.client = createHttpClient(connectionSocketFactory, connectionReuseStrategy);
this.sleeper = sleeper;
}
@@ -227,8 +226,7 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C
}
}
- private static CloseableHttpClient createHttpClient(SSLContext sslContext,
- HostnameVerifier hostnameVerifier,
+ private static CloseableHttpClient createHttpClient(SSLConnectionSocketFactory connectionSocketFactory,
org.apache.http.ConnectionReuseStrategy connectionReuseStrategy) {
RequestConfig config = RequestConfig.custom()
@@ -237,8 +235,7 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C
.setSocketTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()).build();
return HttpClientBuilder.create()
.setUserAgent("config-server-proxy-client")
- .setSSLContext(sslContext)
- .setSSLHostnameVerifier(hostnameVerifier)
+ .setSSLSocketFactory(connectionSocketFactory)
.setDefaultRequestConfig(config)
.setMaxConnPerRoute(10)
.setMaxConnTotal(500)
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java
index c87fea3beb3..4a208aa3794 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java
@@ -5,10 +5,10 @@ import ai.vespa.util.http.hc4.retry.DelayedConnectionLevelRetryHandler;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yahoo.vespa.athenz.api.AthenzIdentity;
-import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier;
import com.yahoo.vespa.flags.FlagId;
import com.yahoo.vespa.flags.json.FlagData;
+import com.yahoo.vespa.hosted.controller.api.integration.ControllerIdentityProvider;
import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget;
import com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire.WireErrorResponse;
import org.apache.http.HttpEntity;
@@ -22,6 +22,7 @@ import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
@@ -55,7 +56,7 @@ class FlagsClient {
private final CloseableHttpClient client;
- FlagsClient(ServiceIdentityProvider identityProvider, Set<FlagsTarget> targets) {
+ FlagsClient(ControllerIdentityProvider identityProvider, Set<FlagsTarget> targets) {
this.client = createClient(identityProvider, targets);
}
@@ -95,14 +96,16 @@ class FlagsClient {
});
}
- private static CloseableHttpClient createClient(ServiceIdentityProvider identityProvider, Set<FlagsTarget> targets) {
+ private static CloseableHttpClient createClient(ControllerIdentityProvider identityProvider, Set<FlagsTarget> targets) {
DelayedConnectionLevelRetryHandler retryHandler = DelayedConnectionLevelRetryHandler.Builder
.withExponentialBackoff(Duration.ofSeconds(1), Duration.ofSeconds(20), 5)
.build();
+ SSLConnectionSocketFactory connectionSocketFactory = new SSLConnectionSocketFactory(
+ identityProvider.getConfigServerSslSocketFactory(), new FlagTargetsHostnameVerifier(targets));
+
return HttpClientBuilder.create()
.setUserAgent("controller-flags-v1-client")
- .setSSLContext(identityProvider.getIdentitySslContext())
- .setSSLHostnameVerifier(new FlagTargetsHostnameVerifier(targets))
+ .setSSLSocketFactory(connectionSocketFactory)
.setDefaultRequestConfig(RequestConfig.custom()
.setConnectTimeout((int) Duration.ofSeconds(10).toMillis())
.setConnectionRequestTimeout((int) Duration.ofSeconds(10).toMillis())
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java
index 1b543045adc..abc888abccb 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java
@@ -4,10 +4,10 @@ package com.yahoo.vespa.hosted.controller.restapi.systemflags;
import com.yahoo.concurrent.DaemonThreadFactory;
import com.yahoo.config.provision.SystemName;
import com.yahoo.text.Text;
-import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
import com.yahoo.vespa.flags.FlagId;
import com.yahoo.vespa.flags.Flags;
import com.yahoo.vespa.flags.json.FlagData;
+import com.yahoo.vespa.hosted.controller.api.integration.ControllerIdentityProvider;
import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget;
import com.yahoo.vespa.hosted.controller.api.systemflags.v1.SystemFlagsDataArchive;
import com.yahoo.vespa.hosted.controller.restapi.systemflags.SystemFlagsDeployResult.OperationError;
@@ -46,7 +46,7 @@ class SystemFlagsDeployer {
private final ExecutorService executor = Executors.newCachedThreadPool(new DaemonThreadFactory("system-flags-deployer-"));
- SystemFlagsDeployer(ServiceIdentityProvider identityProvider, SystemName system, Set<FlagsTarget> targets) {
+ SystemFlagsDeployer(ControllerIdentityProvider identityProvider, SystemName system, Set<FlagsTarget> targets) {
this(new FlagsClient(identityProvider, targets), system, targets);
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java
index aaaf09fa781..ed27ffad978 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java
@@ -8,7 +8,7 @@ import com.yahoo.container.jdisc.ThreadedHttpRequestHandler;
import com.yahoo.restapi.ErrorResponse;
import com.yahoo.restapi.JacksonJsonResponse;
import com.yahoo.restapi.Path;
-import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
+import com.yahoo.vespa.hosted.controller.api.integration.ControllerIdentityProvider;
import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget;
import com.yahoo.vespa.hosted.controller.api.systemflags.v1.SystemFlagsDataArchive;
@@ -30,7 +30,7 @@ public class SystemFlagsHandler extends ThreadedHttpRequestHandler {
@Inject
public SystemFlagsHandler(ZoneRegistry zoneRegistry,
- ServiceIdentityProvider identityProvider,
+ ControllerIdentityProvider identityProvider,
Executor executor) {
super(executor);
this.deployer = new SystemFlagsDeployer(identityProvider, zoneRegistry.system(), FlagsTarget.getAllTargetsInSystem(zoneRegistry, true));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index 0ecac036913..f4f50de59d7 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller;
import com.google.common.collect.Sets;
import com.yahoo.component.Version;
import com.yahoo.config.application.api.DeploymentSpec;
-import com.yahoo.config.application.api.Notifications;
import com.yahoo.config.application.api.ValidationId;
import com.yahoo.config.application.api.ValidationOverrides;
import com.yahoo.config.provision.ApplicationId;
@@ -1177,4 +1176,23 @@ public class ControllerTest {
assertEquals(version2, tester.applications().compileVersion(application, OptionalInt.empty()));
}
+ @Test
+ public void testCloudAccount() {
+ DeploymentContext context = tester.newDeploymentContext();
+ ZoneId zone = ZoneId.from("prod", "us-west-1");
+ String cloudAccount = "012345678912";
+ var applicationPackage = new ApplicationPackageBuilder()
+ .cloudAccount(cloudAccount)
+ .region(zone.region())
+ .build();
+ try {
+ context.submit(applicationPackage).deploy();
+ fail("Expected exception"); // Account invalid for tenant
+ } catch (IllegalArgumentException ignored) {}
+
+ tester.controllerTester().flagSource().withListFlag(PermanentFlags.CLOUD_ACCOUNTS.id(), List.of(cloudAccount), String.class);
+ context.submit(applicationPackage).deploy();
+ assertEquals(cloudAccount, tester.controllerTester().configServer().cloudAccount(context.deploymentIdIn(zone)).get().value());
+ }
+
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
index 27cf1554b4d..5d1a677bf51 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
@@ -66,6 +66,7 @@ public class ApplicationPackageBuilder {
private boolean explicitSystemTest = false;
private boolean explicitStagingTest = false;
private Version compileVersion = Version.fromString("6.1");
+ private String cloudAccount = null;
public ApplicationPackageBuilder majorVersion(int majorVersion) {
this.majorVersion = OptionalInt.of(majorVersion);
@@ -256,13 +257,23 @@ public class ApplicationPackageBuilder {
}
}
+ public ApplicationPackageBuilder cloudAccount(String cloudAccount) {
+ this.cloudAccount = cloudAccount;
+ return this;
+ }
+
private byte[] deploymentSpec() {
StringBuilder xml = new StringBuilder();
xml.append("<deployment version='1.0' ");
majorVersion.ifPresent(v -> xml.append("major-version='").append(v).append("' "));
- if(athenzIdentityAttributes != null) {
+ if (athenzIdentityAttributes != null) {
xml.append(athenzIdentityAttributes);
}
+ if (cloudAccount != null) {
+ xml.append(" cloud-account='");
+ xml.append(cloudAccount);
+ xml.append("'");
+ }
xml.append(">\n");
for (String instance : instances.split(",")) {
xml.append(" <instance id='").append(instance).append("'>\n");
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
index b3689dacea7..35a12f4b6d4 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
@@ -4,10 +4,11 @@ package com.yahoo.vespa.hosted.controller.integration;
import ai.vespa.http.DomainName;
import ai.vespa.http.HttpURL.Path;
import ai.vespa.http.HttpURL.Query;
-import com.yahoo.component.annotation.Inject;
import com.yahoo.component.AbstractComponent;
import com.yahoo.component.Version;
+import com.yahoo.component.annotation.Inject;
import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.CloudAccount;
import com.yahoo.config.provision.ClusterResources;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.DockerImage;
@@ -16,7 +17,6 @@ import com.yahoo.config.provision.HostName;
import com.yahoo.config.provision.NodeResources;
import com.yahoo.config.provision.NodeType;
import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.text.Text;
import com.yahoo.vespa.flags.json.FlagData;
import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics;
import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData;
@@ -49,7 +49,6 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.net.URI;
-import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
@@ -94,6 +93,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
private final Map<DeploymentId, Set<ContainerEndpoint>> containerEndpoints = new HashMap<>();
private final Map<DeploymentId, List<ClusterMetrics>> clusterMetrics = new HashMap<>();
private final Map<DeploymentId, TestReport> testReport = new HashMap<>();
+ private final Map<DeploymentId, CloudAccount> cloudAccounts = new HashMap<>();
private List<ProtonMetrics> protonMetrics;
private Version lastPrepareVersion = null;
@@ -279,6 +279,10 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
return Collections.unmodifiableMap(containerEndpoints);
}
+ public Optional<CloudAccount> cloudAccount(DeploymentId deployment) {
+ return Optional.ofNullable(cloudAccounts.get(deployment));
+ }
+
public Set<String> containerEndpointNames(DeploymentId deployment) {
return containerEndpoints.getOrDefault(deployment, Set.of()).stream()
.map(ContainerEndpoint::names)
@@ -389,6 +393,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
provision(id.zoneId(), id.applicationId(), cluster);
this.containerEndpoints.put(id, deployment.containerEndpoints());
+ deployment.cloudAccount().ifPresent(account -> this.cloudAccounts.put(id, account));
if (!deferLoadBalancerProvisioning.contains(id.zoneId().environment())) {
putLoadBalancers(id.zoneId(), List.of(new LoadBalancer(UUID.randomUUID().toString(),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializerTest.java
index d9e2e61b868..c047f31e171 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializerTest.java
@@ -4,10 +4,10 @@ package com.yahoo.vespa.hosted.controller.persistence;
import com.yahoo.vespa.hosted.controller.auditlog.AuditLog;
import org.junit.Test;
+import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
-import java.util.Optional;
import static java.time.temporal.ChronoUnit.MILLIS;
import static org.junit.Assert.assertEquals;
@@ -28,16 +28,19 @@ public class AuditLogSerializerTest {
AuditLog log = new AuditLog(List.of(
new AuditLog.Entry(i1, "bar", AuditLog.Entry.Method.POST,
"/bar/baz/",
- Optional.of("0".repeat(2048))),
+ "0".repeat(2048).getBytes(StandardCharsets.UTF_8)),
new AuditLog.Entry(i2, "foo", AuditLog.Entry.Method.POST,
"/foo/bar/",
- Optional.of("{\"foo\":\"bar\"}")),
+ "{\"foo\":\"bar\"}".getBytes(StandardCharsets.UTF_8)),
new AuditLog.Entry(i3, "baz", AuditLog.Entry.Method.POST,
"/foo/baz/",
- Optional.of("")),
+ new byte[0]),
new AuditLog.Entry(i4, "baz", AuditLog.Entry.Method.POST,
"/foo/baz/",
- Optional.of("\ufdff\ufeff\uffff")) // non-ascii
+ "000\ufdff\ufeff\uffff000".getBytes(StandardCharsets.UTF_8)), // non-ascii
+ new AuditLog.Entry(i4, "quux", AuditLog.Entry.Method.POST,
+ "/foo/quux/",
+ new byte[]{(byte) 0xDE, (byte) 0xAD, (byte) 0xBE, (byte) 0xEF}) // garbage
));
AuditLogSerializer serializer = new AuditLogSerializer();
@@ -58,6 +61,7 @@ public class AuditLogSerializerTest {
assertEquals(1024, log.entries().get(0).data().get().length());
assertTrue(log.entries().get(2).data().isEmpty());
assertTrue(log.entries().get(3).data().isEmpty());
+ assertTrue(log.entries().get(4).data().isEmpty());
}
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java
index c4fbf1aa3a5..f5926e799af 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java
@@ -4,11 +4,10 @@ package com.yahoo.vespa.hosted.controller.proxy;
import ai.vespa.http.HttpURL.Path;
import com.github.tomakehurst.wiremock.junit.WireMockRule;
import com.github.tomakehurst.wiremock.stubbing.Scenario;
-import com.yahoo.config.provision.SystemName;
import com.yahoo.container.jdisc.HttpRequest;
import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock;
import com.yahoo.yolean.concurrent.Sleeper;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.protocol.HttpContext;
import org.apache.http.protocol.HttpCoreContext;
import org.junit.Rule;
@@ -39,7 +38,7 @@ public class ConfigServerRestExecutorImplTest {
@Test
public void proxy_with_retries() throws Exception {
var connectionReuseStrategy = new CountingConnectionReuseStrategy(Set.of("127.0.0.1"));
- var proxy = new ConfigServerRestExecutorImpl(new ZoneRegistryMock(SystemName.cd), SSLContext.getDefault(),
+ var proxy = new ConfigServerRestExecutorImpl(new SSLConnectionSocketFactory(SSLContext.getDefault()),
Sleeper.NOOP, connectionReuseStrategy);
URI url = url();
@@ -64,8 +63,8 @@ public class ConfigServerRestExecutorImplTest {
@Test
public void proxy_without_connection_reuse() throws Exception {
var connectionReuseStrategy = new CountingConnectionReuseStrategy(Set.of());
- var proxy = new ConfigServerRestExecutorImpl(new ZoneRegistryMock(SystemName.cd), SSLContext.getDefault(),
- (duration) -> {}, connectionReuseStrategy);
+ var proxy = new ConfigServerRestExecutorImpl(new SSLConnectionSocketFactory(SSLContext.getDefault()),
+ Sleeper.NOOP, connectionReuseStrategy);
URI url = url();
String path = url.getPath();