diff options
author | Jon Marius Venstad <venstad@gmail.com> | 2020-09-01 17:22:43 +0200 |
---|---|---|
committer | Jon Marius Venstad <venstad@gmail.com> | 2020-09-01 17:22:43 +0200 |
commit | 30f02e92849dcdb1b1de17810d46a761eb03bc19 (patch) | |
tree | a76f0c4a2f3e38c9c8d9bd48099c748790ded66f /controller-server/src | |
parent | caf58f5b7ed8cdda43ec73706986b9e41dd019ed (diff) |
Add and use API for storing deployment meeta data
Diffstat (limited to 'controller-server/src')
4 files changed, 67 insertions, 14 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 05ff2c92d85..83a1895a85d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -613,6 +613,7 @@ public class ApplicationController { applicationStore.removeAll(id.tenant(), id.application()); applicationStore.removeAllTesters(id.tenant(), id.application()); + applicationStore.putMetaTombstone(id.tenant(), id.application(), clock.instant()); accessControl.deleteApplication(id, credentials); curator.removeApplication(id); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java index d0e8a0f2816..b246e10ca82 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java @@ -26,11 +26,15 @@ import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.nio.charset.StandardCharsets.UTF_8; @@ -46,11 +50,16 @@ import static java.nio.charset.StandardCharsets.UTF_8; public class ApplicationPackage { private static final String trustedCertificatesFile = "security/clients.pem"; + private static final String buildMetaFile = "build-meta.json"; + private static final String deploymentFile = "deployment.xml"; + private static final String validationOverridesFile = "validation-overrides.xml"; + private static final String servicesFile = "services.xml"; private final String contentHash; private final byte[] zippedContent; private final DeploymentSpec deploymentSpec; private final ValidationOverrides validationOverrides; + private final Files files; private final Optional<Version> compileVersion; private final Optional<Instant> buildTime; private final List<X509Certificate> trustedCertificates; @@ -73,18 +82,18 @@ public class ApplicationPackage { public ApplicationPackage(byte[] zippedContent, boolean requireFiles) { this.zippedContent = Objects.requireNonNull(zippedContent, "The application package content cannot be null"); this.contentHash = Hashing.sha1().hashBytes(zippedContent).toString(); - Files files = Files.extract(Set.of("deployment.xml", "validation-overrides.xml", "build-meta.json", trustedCertificatesFile), zippedContent); + this.files = Files.extract(Set.of(deploymentFile, validationOverridesFile, servicesFile, buildMetaFile, trustedCertificatesFile), zippedContent); - Optional<DeploymentSpec> deploymentSpec = files.getAsReader("deployment.xml").map(DeploymentSpec::fromXml); + Optional<DeploymentSpec> deploymentSpec = files.getAsReader(deploymentFile).map(DeploymentSpec::fromXml); if (requireFiles && deploymentSpec.isEmpty()) - throw new IllegalArgumentException("Missing required file 'deployment.xml'"); + throw new IllegalArgumentException("Missing required file '" + deploymentFile + "'"); this.deploymentSpec = deploymentSpec.orElse(DeploymentSpec.empty); - this.validationOverrides = files.getAsReader("validation-overrides.xml").map(ValidationOverrides::fromXml).orElse(ValidationOverrides.empty); + this.validationOverrides = files.getAsReader(validationOverridesFile).map(ValidationOverrides::fromXml).orElse(ValidationOverrides.empty); - Optional<Inspector> buildMetaObject = files.get("build-meta.json").map(SlimeUtils::jsonToSlime).map(Slime::get); + Optional<Inspector> buildMetaObject = files.get(buildMetaFile).map(SlimeUtils::jsonToSlime).map(Slime::get); if (requireFiles && buildMetaObject.isEmpty()) - throw new IllegalArgumentException("Missing required file 'build-meta.json'"); + throw new IllegalArgumentException("Missing required file '" + buildMetaFile + "'"); this.compileVersion = buildMetaObject.flatMap(object -> parse(object, "compileVersion", field -> Version.fromString(field.asString()))); this.buildTime = buildMetaObject.flatMap(object -> parse(object, "buildTime", field -> Instant.ofEpochMilli(field.asLong()))); @@ -133,12 +142,12 @@ public class ApplicationPackage { private static <Type> Optional<Type> parse(Inspector buildMetaObject, String fieldName, Function<Inspector, Type> mapper) { if ( ! buildMetaObject.field(fieldName).valid()) - throw new IllegalArgumentException("Missing value '" + fieldName + "' in 'build-meta.json'"); + throw new IllegalArgumentException("Missing value '" + fieldName + "' in '" + buildMetaFile + "'"); try { return Optional.of(mapper.apply(buildMetaObject.field(fieldName))); } catch (RuntimeException e) { - throw new IllegalArgumentException("Failed parsing \"" + fieldName + "\" in 'build-meta.json': " + Exceptions.toMessageString(e)); + throw new IllegalArgumentException("Failed parsing \"" + fieldName + "\" in '" + buildMetaFile + "': " + Exceptions.toMessageString(e)); } } @@ -191,14 +200,23 @@ public class ApplicationPackage { /** Creates a valid application package that will remove all application's deployments */ public static ApplicationPackage deploymentRemoval() { - DeploymentSpec deploymentSpec = DeploymentSpec.empty; - ValidationOverrides validationOverrides = allValidationOverrides(); - try (ZipBuilder zipBuilder = new ZipBuilder(deploymentSpec.xmlForm().length() + validationOverrides.xmlForm().length() + 500)) { - zipBuilder.add("validation-overrides.xml", validationOverrides.xmlForm().getBytes(UTF_8)); - zipBuilder.add("deployment.xml", deploymentSpec.xmlForm().getBytes(UTF_8)); + return new ApplicationPackage(filesZip(Map.of(validationOverridesFile, allValidationOverrides().xmlForm().getBytes(UTF_8), + deploymentFile, DeploymentSpec.empty.xmlForm().getBytes(UTF_8)))); + } + + /** Returns a zip containing only services.xml and deployment.xml files of this. */ + public byte[] metaDataZip() { + return filesZip(Stream.of(deploymentFile, servicesFile) + .filter(name -> files.files.containsKey(name)) + .collect(Collectors.toMap(name -> name, + name -> files.files.get(name)))); + } + private static byte[] filesZip(Map<String, byte[]> files) { + try (ZipBuilder zipBuilder = new ZipBuilder(files.values().stream().mapToInt(bytes -> bytes.length).sum() + 512)) { + files.forEach(zipBuilder::add); zipBuilder.close(); - return new ApplicationPackage(zipBuilder.toByteArray()); + return zipBuilder.toByteArray(); } } @@ -212,4 +230,5 @@ public class ApplicationPackage { return ValidationOverrides.fromXml(validationOverridesContents.toString()); } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 5fcb3f30ae9..3bc9bc01a76 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -414,6 +414,10 @@ public class JobController { id.application(), version.get(), testPackageBytes); + controller.applications().applicationStore().putMeta(id.tenant(), + id.application(), + controller.clock().instant(), + applicationPackage.metaDataZip()); prunePackages(id); controller.applications().storeWithUpdatedConfig(application, applicationPackage); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java index 3c33051e98b..be0cd975190 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java @@ -10,9 +10,13 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationS import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; +import java.time.Instant; import java.util.Map; +import java.util.NavigableMap; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListMap; import static java.util.Objects.requireNonNull; @@ -23,8 +27,11 @@ import static java.util.Objects.requireNonNull; */ public class ApplicationStoreMock implements ApplicationStore { + private static final byte[] tombstone = new byte[0]; + private final Map<ApplicationId, Map<ApplicationVersion, byte[]>> store = new ConcurrentHashMap<>(); private final Map<ApplicationId, Map<ZoneId, byte[]>> devStore = new ConcurrentHashMap<>(); + private final Map<ApplicationId, NavigableMap<Instant, byte[]>> meta = new ConcurrentHashMap<>(); private static ApplicationId appId(TenantName tenant, ApplicationName application) { return ApplicationId.from(tenant, application, InstanceName.defaultName()); @@ -101,4 +108,26 @@ public class ApplicationStoreMock implements ApplicationStore { return requireNonNull(devStore.get(application).get(zone)); } + @Override + public void putMeta(TenantName tenant, ApplicationName application, Instant now, byte[] metaZip) { + meta.putIfAbsent(appId(tenant, application), new ConcurrentSkipListMap<>()); + meta.get(appId(tenant, application)).put(now, metaZip); + } + + @Override + public void putMetaTombstone(TenantName tenant, ApplicationName application, Instant now) { + putMeta(tenant, application, now, tombstone); + } + + @Override + public void pruneMeta(Instant oldest) { + for (ApplicationId id : meta.keySet()) { + Instant activeAtOldest = meta.get(id).lowerKey(oldest); + if (activeAtOldest != null) + meta.get(id).headMap(activeAtOldest).clear(); + if (meta.get(id).lastKey().isBefore(oldest) && meta.get(id).lastEntry().getValue() == tombstone) + meta.remove(id); + } + } + } |