diff options
Diffstat (limited to 'controller-server/src/test/java/com')
43 files changed, 444 insertions, 92 deletions
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 86b9370150c..433a976358e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -26,7 +26,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedAliasTarget; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.Endpoint; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiffTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiffTest.java new file mode 100644 index 00000000000..b2aba721a6f --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiffTest.java @@ -0,0 +1,128 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application.pkg; + +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Map; +import java.util.zip.Deflater; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageDiff.diff; + +import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageDiff.diffAgainstEmpty; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertEquals; + +/** + * @author freva + */ +public class ApplicationPackageDiffTest { + private static final ApplicationPackage app1 = applicationPackage(Map.of("file1", "contents of the\nfirst file", "dir/myfile", "Second file", "dir/binary", "øøøø")); + private static final ApplicationPackage app2 = applicationPackage(Map.of("file1", "updated contents\nof the\nfirst file\nafter some changes", "dir/myfile2", "Second file", "dir/binary", "øøøø")); + + @Test + public void no_diff() { + assertEquals("No diff\n", new String(diff(app1, app1))); + } + + @Test + public void diff_against_empty() { + assertEquals("--- dir/binary\n" + + "Diff skipped: File is binary (new file -> 8B)\n" + + "\n" + + "--- dir/myfile\n" + + "@@ -1,0 +1,1 @@\n" + + "+ Second file\n" + + "\n" + + "--- file1\n" + + "@@ -1,0 +1,2 @@\n" + + "+ contents of the\n" + + "+ first file\n" + + "\n", new String(diffAgainstEmpty(app1))); + } + + @Test + public void full_diff() { + // Even though dir/binary is binary file, we can see they are identical, so it should not print "Diff skipped" + assertEquals("--- dir/myfile\n" + + "@@ -1,1 +1,0 @@\n" + + "- Second file\n" + + "\n" + + "--- dir/myfile2\n" + + "@@ -1,0 +1,1 @@\n" + + "+ Second file\n" + + "\n" + + "--- file1\n" + + "@@ -1,2 +1,4 @@\n" + + "+ updated contents\n" + + "+ of the\n" + + "- contents of the\n" + + " first file\n" + + "+ after some changes\n" + + "\n", new String(diff(app1, app2))); + } + + @Test + public void skips_diff_for_too_large_files() { + assertEquals("--- dir/myfile\n" + + "@@ -1,1 +1,0 @@\n" + + "- Second file\n" + + "\n" + + "--- dir/myfile2\n" + + "@@ -1,0 +1,1 @@\n" + + "+ Second file\n" + + "\n" + + "--- file1\n" + + "Diff skipped: File too large (26B -> 53B)\n" + + "\n", new String(diff(app1, app2, 12, 1000, 1000))); + } + + @Test + public void skips_diff_if_file_diff_is_too_large() { + assertEquals("--- dir/myfile\n" + + "@@ -1,1 +1,0 @@\n" + + "- Second file\n" + + "\n" + + "--- dir/myfile2\n" + + "@@ -1,0 +1,1 @@\n" + + "+ Second file\n" + + "\n" + + "--- file1\n" + + "Diff skipped: Diff too large (96B)\n" + + "\n", new String(diff(app1, app2, 1000, 50, 1000))); + } + + @Test + public void skips_diff_if_total_diff_is_too_large() { + assertEquals("--- dir/myfile\n" + + "@@ -1,1 +1,0 @@\n" + + "- Second file\n" + + "\n" + + "--- dir/myfile2\n" + + "Diff skipped: Total diff size >20B)\n" + + "\n" + + "--- file1\n" + + "Diff skipped: Total diff size >20B)\n" + + "\n", new String(diff(app1, app2, 1000, 1000, 20))); + } + + private static ApplicationPackage applicationPackage(Map<String, String> files) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZipOutputStream out = new ZipOutputStream(baos)) { + out.setLevel(Deflater.NO_COMPRESSION); // This is for testing purposes so we skip compression for performance + for (Map.Entry<String, String> file : files.entrySet()) { + ZipEntry entry = new ZipEntry(file.getKey()); + out.putNextEntry(entry); + out.write(file.getValue().getBytes(UTF_8)); + out.closeEntry(); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return new ApplicationPackage(baos.toByteArray()); + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java index 1849be9b6bd..75e00e3434c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java @@ -1,5 +1,5 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; +package com.yahoo.vespa.hosted.controller.application.pkg; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; @@ -109,10 +109,10 @@ public class ApplicationPackageTest { } private static Map<String, String> unzip(byte[] zip) { - return new ZipStreamReader(new ByteArrayInputStream(zip), __ -> true, 1 << 10) + return new ZipStreamReader(new ByteArrayInputStream(zip), __ -> true, 1 << 10, true) .entries().stream() .collect(Collectors.toMap(entry -> entry.zipEntry().getName(), - entry -> new String(entry.content(), UTF_8))); + entry -> new String(entry.contentOrThrow(), UTF_8))); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/LinesComparatorTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/LinesComparatorTest.java new file mode 100644 index 00000000000..92137094f62 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/LinesComparatorTest.java @@ -0,0 +1,112 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application.pkg; + +import org.junit.Test; + +import java.util.Optional; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; + +public class LinesComparatorTest { + private static final String text1 = "This part of the\n" + + "document has stayed the\n" + + "same from version to\n" + + "version. It shouldn't\n" + + "be shown if it doesn't\n" + + "change. Otherwise, that\n" + + "would not be helping to\n" + + "compress the size of the\n" + + "changes.\n" + + "\n" + + "This paragraph contains\n" + + "text that is outdated.\n" + + "It will be deleted in the\n" + + "near future.\n" + + "\n" + + "It is important to spell\n" + + "check this dokument. On\n" + + "the other hand, a\n" + + "misspelled word isn't\n" + + "the end of the world.\n" + + "Nothing in the rest of\n" + + "this paragraph needs to\n" + + "be changed. Things can\n" + + "be added after it."; + private static final String text2 = "This is an important\n" + + "notice! It should\n" + + "therefore be located at\n" + + "the beginning of this\n" + + "document!\n" + + "\n" + + "This part of the\n" + + "document has stayed the\n" + + "same from version to\n" + + "version. It shouldn't\n" + + "be shown if it doesn't\n" + + "change. Otherwise, that\n" + + "would not be helping to\n" + + "compress the size of the\n" + + "changes.\n" + + "\n" + + "It is important to spell\n" + + "check this document. On\n" + + "the other hand, a\n" + + "misspelled word isn't\n" + + "the end of the world.\n" + + "Nothing in the rest of\n" + + "this paragraph needs to\n" + + "be changed. Things can\n" + + "be added after it.\n" + + "\n" + + "This paragraph contains\n" + + "important new additions\n" + + "to this document."; + + @Test + public void diff_test() { + assertDiff(null, "", ""); + assertDiff(null, text1, text1); + assertDiff(text1.lines().map(line -> "- " + line).collect(Collectors.joining("\n", "@@ -1,24 +1,0 @@\n", "\n")), text1, ""); + assertDiff(text1.lines().map(line -> "+ " + line).collect(Collectors.joining("\n", "@@ -1,0 +1,24 @@\n", "\n")), "", text1); + assertDiff("@@ -1,3 +1,9 @@\n" + + "+ This is an important\n" + + "+ notice! It should\n" + + "+ therefore be located at\n" + + "+ the beginning of this\n" + + "+ document!\n" + + "+ \n" + + " This part of the\n" + + " document has stayed the\n" + + " same from version to\n" + + "@@ -7,14 +13,9 @@\n" + + " would not be helping to\n" + + " compress the size of the\n" + + " changes.\n" + + "- \n" + + "- This paragraph contains\n" + + "- text that is outdated.\n" + + "- It will be deleted in the\n" + + "- near future.\n" + + " \n" + + " It is important to spell\n" + + "+ check this document. On\n" + + "- check this dokument. On\n" + + " the other hand, a\n" + + " misspelled word isn't\n" + + " the end of the world.\n" + + "@@ -22,3 +23,7 @@\n" + + " this paragraph needs to\n" + + " be changed. Things can\n" + + " be added after it.\n" + + "+ \n" + + "+ This paragraph contains\n" + + "+ important new additions\n" + + "+ to this document.\n", text1, text2); + } + + private static void assertDiff(String expected, String left, String right) { + assertEquals(Optional.ofNullable(expected), + LinesComparator.diff(left.lines().collect(Collectors.toList()), right.lines().collect(Collectors.toList()))); + } +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ZipStreamReaderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReaderTest.java index abd234f0fa4..afbd232f01c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ZipStreamReaderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReaderTest.java @@ -1,5 +1,5 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; +package com.yahoo.vespa.hosted.controller.application.pkg; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; @@ -38,15 +38,15 @@ public class ZipStreamReaderTest { public void test_size_limit() { Map<String, String> entries = Map.of("foo.xml", "foobar"); try { - new ZipStreamReader(new ByteArrayInputStream(zip(entries)), "foo.xml"::equals, 1); + new ZipStreamReader(new ByteArrayInputStream(zip(entries)), "foo.xml"::equals, 1, true); fail("Expected exception"); } catch (IllegalArgumentException ignored) {} entries = Map.of("foo.xml", "foobar", "foo.jar", "0".repeat(100) // File not extracted and thus not subject to size limit ); - ZipStreamReader reader = new ZipStreamReader(new ByteArrayInputStream(zip(entries)), "foo.xml"::equals,10); - byte[] extracted = reader.entries().get(0).content(); + ZipStreamReader reader = new ZipStreamReader(new ByteArrayInputStream(zip(entries)), "foo.xml"::equals, 10, true); + byte[] extracted = reader.entries().get(0).contentOrThrow(); assertEquals("foobar", new String(extracted, StandardCharsets.UTF_8)); } @@ -65,7 +65,7 @@ public class ZipStreamReaderTest { ); tests.forEach((name, expectException) -> { try { - new ZipStreamReader(new ByteArrayInputStream(zip(Map.of(name, "foo"))), name::equals, 1024); + new ZipStreamReader(new ByteArrayInputStream(zip(Map.of(name, "foo"))), name::equals, 1024, true); assertFalse("Expected exception for '" + name + "'", expectException); } catch (IllegalArgumentException ignored) { assertTrue("Unexpected exception for '" + name + "'", expectException); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index b234ab4960b..73b1489b088 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -10,7 +10,7 @@ import com.yahoo.security.SignatureAlgorithm; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.X509CertificateUtils; import com.yahoo.text.Text; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import javax.security.auth.x500.X500Principal; import java.io.ByteArrayOutputStream; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 420d0be04ac..c225dcbe49d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -28,7 +28,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; @@ -492,8 +492,8 @@ public class DeploymentContext { Run run = jobs.last(job) .filter(r -> r.id().type() == job.type()) .orElseThrow(() -> new AssertionError(job.type() + " is not among the active: " + jobs.active())); - assertFalse(run.id() + " should not have failed yet", run.hasFailed()); - assertFalse(run.id() + " should not have ended yet", run.hasEnded()); + assertFalse(run.id() + " should not have failed yet: " + run, run.hasFailed()); + assertFalse(run.id() + " should not have ended yet: " + run, run.hasEnded()); return run; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index d4c9425fc03..ad32266e290 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -8,7 +8,7 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import org.junit.Assert; @@ -61,7 +61,7 @@ import static org.junit.Assert.assertTrue; */ public class DeploymentTriggerTest { - private DeploymentTester tester = new DeploymentTester(); + private final DeploymentTester tester = new DeploymentTester(); @Test public void testTriggerFailing() { @@ -1110,15 +1110,16 @@ public class DeploymentTriggerTest { // System and staging tests both require unknown versions, and are broken. tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsCentral1, "user", false); app.runJob(productionCdUsCentral1) - .abortJob(systemTest) - .abortJob(stagingTest) + .jobAborted(systemTest) + .jobAborted(stagingTest) .runJob(systemTest) .runJob(stagingTest) .runJob(productionCdAwsUsEast1a); app.runJob(productionCdUsCentral1, cdPackage); app.submit(cdPackage); - app.runJob(systemTest); + app.jobAborted(systemTest) + .runJob(systemTest); // Staging test requires unknown initial version, and is broken. tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsCentral1, "user", false); app.runJob(productionCdUsCentral1) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index d685c6a2354..780d2d226f3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -25,7 +25,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.config.ControllerConfig; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java index 59e2b6c04d8..521ff160a05 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java @@ -5,11 +5,11 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import java.time.Instant; import java.util.Map; @@ -30,7 +30,9 @@ public class ApplicationStoreMock implements ApplicationStore { private static final byte[] tombstone = new byte[0]; private final Map<ApplicationId, Map<ApplicationVersion, byte[]>> store = new ConcurrentHashMap<>(); - private final Map<ApplicationId, Map<ZoneId, byte[]>> devStore = new ConcurrentHashMap<>(); + private final Map<DeploymentId, byte[]> devStore = new ConcurrentHashMap<>(); + private final Map<ApplicationId, Map<Long, byte[]>> diffs = new ConcurrentHashMap<>(); + private final Map<DeploymentId, Map<Long, byte[]>> devDiffs = new ConcurrentHashMap<>(); private final Map<ApplicationId, NavigableMap<Instant, byte[]>> meta = new ConcurrentHashMap<>(); private final Map<DeploymentId, NavigableMap<Instant, byte[]>> metaManual = new ConcurrentHashMap<>(); @@ -43,15 +45,30 @@ public class ApplicationStoreMock implements ApplicationStore { } @Override - public byte[] get(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion) { - byte[] bytes = store.get(appId(tenant, application)).get(applicationVersion); + public byte[] get(DeploymentId deploymentId, ApplicationVersion applicationVersion) { + if (applicationVersion.isDeployedDirectly()) + return requireNonNull(devStore.get(deploymentId)); + + TenantAndApplicationId tenantAndApplicationId = TenantAndApplicationId.from(deploymentId.applicationId()); + byte[] bytes = store.get(appId(tenantAndApplicationId.tenant(), tenantAndApplicationId.application())).get(applicationVersion); if (bytes == null) - throw new IllegalArgumentException("No application package found for " + tenant + "." + application + + throw new IllegalArgumentException("No application package found for " + tenantAndApplicationId + " with version " + applicationVersion.id()); return bytes; } @Override + public Optional<byte[]> getDiff(TenantName tenantName, ApplicationName applicationName, long buildNumber) { + return Optional.ofNullable(diffs.get(appId(tenantName, applicationName))).map(map -> map.get(buildNumber)); + } + + @Override + public void pruneDiffs(TenantName tenantName, ApplicationName applicationName, long beforeBuildNumber) { + Optional.ofNullable(diffs.get(appId(tenantName, applicationName))) + .ifPresent(map -> map.keySet().removeIf(buildNumber -> buildNumber < beforeBuildNumber)); + } + + @Override public Optional<byte[]> find(TenantName tenant, ApplicationName application, long buildNumber) { return store.getOrDefault(appId(tenant, application), Map.of()).entrySet().stream() .filter(kv -> kv.getKey().buildNumber().orElse(Long.MIN_VALUE) == buildNumber) @@ -60,9 +77,10 @@ public class ApplicationStoreMock implements ApplicationStore { } @Override - public void put(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion, byte[] applicationPackage) { - store.putIfAbsent(appId(tenant, application), new ConcurrentHashMap<>()); - store.get(appId(tenant, application)).put(applicationVersion, applicationPackage); + public void put(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion, byte[] applicationPackage, byte[] diff) { + store.computeIfAbsent(appId(tenant, application), __ -> new ConcurrentHashMap<>()).put(applicationVersion, applicationPackage); + applicationVersion.buildNumber().ifPresent(buildNumber -> + diffs.computeIfAbsent(appId(tenant, application), __ -> new ConcurrentHashMap<>()).put(buildNumber, diff)); } @Override @@ -83,8 +101,8 @@ public class ApplicationStoreMock implements ApplicationStore { @Override public void putTester(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion, byte[] testerPackage) { - store.putIfAbsent(testerId(tenant, application), new ConcurrentHashMap<>()); - store.get(testerId(tenant, application)).put(applicationVersion, testerPackage); + store.computeIfAbsent(testerId(tenant, application), key -> new ConcurrentHashMap<>()) + .put(applicationVersion, testerPackage); } @Override @@ -99,14 +117,21 @@ public class ApplicationStoreMock implements ApplicationStore { } @Override - public void putDev(ApplicationId application, ZoneId zone, byte[] applicationPackage) { - devStore.putIfAbsent(application, new ConcurrentHashMap<>()); - devStore.get(application).put(zone, applicationPackage); + public Optional<byte[]> getDevDiff(DeploymentId deploymentId, long buildNumber) { + return Optional.ofNullable(devDiffs.get(deploymentId)).map(map -> map.get(buildNumber)); + } + + @Override + public void pruneDevDiffs(DeploymentId deploymentId, long beforeBuildNumber) { + Optional.ofNullable(devDiffs.get(deploymentId)) + .ifPresent(map -> map.keySet().removeIf(buildNumber -> buildNumber < beforeBuildNumber)); } @Override - public byte[] getDev(ApplicationId application, ZoneId zone) { - return requireNonNull(devStore.get(application).get(zone)); + public void putDev(DeploymentId deploymentId, ApplicationVersion applicationVersion, byte[] applicationPackage, byte[] diff) { + devStore.put(deploymentId, applicationPackage); + applicationVersion.buildNumber().ifPresent(buildNumber -> + devDiffs.computeIfAbsent(deploymentId, __ -> new ConcurrentHashMap<>()).put(buildNumber, diff)); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index e18542108c0..cbdf5dcb075 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -40,7 +40,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.vespa.serviceview.bindings.ClusterView; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java index d7934f08fee..bd50078bc87 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java @@ -9,7 +9,7 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java index 31f8aaf9e2d..b27e30e9ed6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java index af2c0c6b08d..7ee1349f6d5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java @@ -7,7 +7,7 @@ import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.stubs.LoggingDeploymentIssues; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java index 8083b847c0b..0cb7b192b8b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java @@ -9,7 +9,7 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java index 7a8f775e8b1..ec33c8a7048 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java @@ -5,21 +5,13 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; -import com.yahoo.vespa.hosted.controller.deployment.Run; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import org.junit.Test; import java.time.Duration; import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.time.temporal.TemporalUnit; -import java.util.Optional; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.devUsEast1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 023c5671b60..7970e20f6c7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -7,7 +7,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.JobController; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index a9046a8e060..2fb5aee354b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -15,7 +15,7 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java index 00d39788e38..9e8842243c0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java index 5ef64b460b9..a255a6c37d8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java @@ -12,7 +12,7 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.MetricsMock; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainerTest.java index df93efab893..3fd9afe5445 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainerTest.java @@ -5,7 +5,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.RetriggerEntry; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java index f4e688379e5..3c99034761c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockRoleService; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import org.hamcrest.Matchers; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java index 29d77c38b1a..f2f71b9a5b8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java @@ -7,7 +7,7 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; import org.junit.Test; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 91b5dc232e5..1dd4ef24c9e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 37a173ffc37..2ae45c75cae 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -88,7 +88,8 @@ public class ApplicationSerializerTest { Optional.of(Version.fromString("1.2.3")), Optional.of(Instant.ofEpochMilli(666)), Optional.empty(), - Optional.of("best commit")); + Optional.of("best commit"), + true); assertEquals("https://github/org/repo/tree/commit1", applicationVersion1.sourceUrl().get()); ApplicationVersion applicationVersion2 = ApplicationVersion diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index 03a050db74e..7623a02f6af 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -82,13 +82,16 @@ public class RunSerializerTest { assertEquals(running, run.status()); assertEquals(3, run.lastTestLogEntry()); assertEquals(new Version(1, 2, 3), run.versions().targetPlatform()); - ApplicationVersion applicationVersion = ApplicationVersion.from(new SourceRevision("git@github.com:user/repo.git", - "master", - "f00bad"), + ApplicationVersion applicationVersion = ApplicationVersion.from(Optional.of(new SourceRevision("git@github.com:user/repo.git", + "master", + "f00bad")), 123, - "a@b", - Version.fromString("6.3.1"), - Instant.ofEpochMilli(100)); + Optional.of("a@b"), + Optional.of(Version.fromString("6.3.1")), + Optional.of(Instant.ofEpochMilli(100)), + Optional.empty(), + Optional.empty(), + true); assertEquals(applicationVersion, run.versions().targetApplication()); assertEquals(applicationVersion.authorEmail(), run.versions().targetApplication().authorEmail()); assertEquals(applicationVersion.buildTime(), run.versions().targetApplication().buildTime()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json index 0f40dd27664..7b9131a38dd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json @@ -40,6 +40,7 @@ "branch": "master", "commit": "f00bad", "build": 123, + "deployedDirectly": true, "authorEmail": "a@b", "compileVersion": "6.3.1", "buildTime": 100, @@ -48,7 +49,8 @@ "repository": "git@github.com:user/repo.git", "branch": "master", "commit": "badb17", - "build": 122 + "build": 122, + "deployedDirectly": false } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 9f366aa25e0..bf2cd039afd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -52,7 +52,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringData; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; @@ -300,6 +300,13 @@ public class ApplicationApiTest extends ControllerContainerTest { .data(createApplicationDeployData(applicationPackageInstance1, false)), new File("deployment-job-accepted-2.json")); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/myuser/job/dev-us-east-1/diff/1", GET).userIdentity(HOSTED_VESPA_OPERATOR), + (response) -> assertTrue(response.getBodyAsString(), + response.getBodyAsString().contains("--- search-definitions/test.sd\n" + + "@@ -1,0 +1,1 @@\n" + + "+ search test { }\n")), + 200); + // DELETE a dev deployment is allowed under user instance for tenant admins tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/myuser/environment/dev/region/us-east-1", DELETE) .userIdentity(USER_ID), @@ -757,6 +764,12 @@ public class ApplicationApiTest extends ControllerContainerTest { .data(createApplicationSubmissionData(packageWithService, 123)), "{\"message\":\"Application package version: 1.0.2-commit1, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/diff/2", GET).userIdentity(HOSTED_VESPA_OPERATOR), + (response) -> assertTrue(response.getBodyAsString(), + response.getBodyAsString().contains("+ <deployment version='1.0' athenz-domain='domain1' athenz-service='service'>\n" + + "- <deployment version='1.0' >\n")), + 200); + // GET last submitted application package tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package", GET).userIdentity(HOSTED_VESPA_OPERATOR), (response) -> { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index 72295497c03..de5ae466039 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -9,7 +9,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import org.junit.Test; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index 2813bd0ab7d..abe3d4100d9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -502,7 +502,9 @@ "status": "success", "versions": { "targetPlatform": "6.1.0", - "targetApplication": {} + "targetApplication": { + "build": 1 + } }, "steps": [ { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json index acde58f2a28..dce73ad56cd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json @@ -8,7 +8,9 @@ "status": "success", "versions": { "targetPlatform": "7.1.0", - "targetApplication": {} + "targetApplication": { + "build": 1 + } }, "steps": [ { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json index e3beb371acd..92a823bdfc2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json @@ -5,7 +5,9 @@ "runs": [ { "versions": { - "targetApplication": {}, + "targetApplication": { + "build": 1 + }, "targetPlatform": "6.1.0" }, "start": 0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json index 72411d155c7..3ef993c6589 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json @@ -6,7 +6,7 @@ { "at": 0, "type": "info", - "message": "Deploying platform version 6.1 and application version unknown ..." + "message": "Deploying platform version 6.1 and application version 1.0.1 ..." }, { "at": 0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json index 9a742a9b176..7ebc2d24fe9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json @@ -11,7 +11,9 @@ "status": "success", "versions": { "targetPlatform": "6.1.0", - "targetApplication": {} + "targetApplication": { + "build": 1 + } }, "steps": [ { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json index 2601937faee..f8aba54356b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json @@ -5,7 +5,9 @@ "runs": [ { "versions": { - "targetApplication": {}, + "targetApplication": { + "build": 1 + }, "targetPlatform": "7.1.0" }, "start": 14503000, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java index 63474ebb7c9..438da66e6e8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.restapi.deployment; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index cd24ec170c5..460afb102d9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -7,7 +7,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiTest.java new file mode 100644 index 00000000000..8e51f8210c7 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiTest.java @@ -0,0 +1,67 @@ +package com.yahoo.vespa.hosted.controller.restapi.horizon; + +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; +import com.yahoo.vespa.hosted.controller.api.role.Role; +import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; +import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerCloudTest; +import org.junit.Test; + +import java.util.Set; + +/** + * @author olaa + */ +public class HorizonApiTest extends ControllerContainerCloudTest { + + @Test + public void only_operators_and_flag_enabled_tenants_allowed() { + ContainerTester tester = new ContainerTester(container, ""); + TenantName tenantName = TenantName.defaultName(); + + tester.assertResponse(request("/horizon/v1/config/dashboard/topFolders") + .roles(Set.of(Role.hostedOperator())), + "", 200); + + tester.assertResponse(request("/horizon/v1/config/dashboard/topFolders") + .roles(Set.of(Role.reader(tenantName))), + "{\"error-code\":\"FORBIDDEN\",\"message\":\"No tenant with enabled metrics view\"}", 403); + + ((InMemoryFlagSource) tester.controller().flagSource()) + .withBooleanFlag(Flags.ENABLED_HORIZON_DASHBOARD.id(), true); + + tester.controller().serviceRegistry().billingController().setPlan(tenantName, PlanId.from("pay-as-you-go"), true); + + tester.assertResponse(request("/horizon/v1/config/dashboard/topFolders") + .roles(Set.of(Role.reader(tenantName))), + "", 200); + } + + @Override + protected SystemName system() { + return SystemName.PublicCd; + } + + @Override + protected String variablePartXml() { + return " <component id='com.yahoo.vespa.hosted.controller.security.CloudAccessControlRequests'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.security.CloudAccessControl'/>\n" + + + " <handler id=\"com.yahoo.vespa.hosted.controller.restapi.horizon.HorizonApiHandler\" bundle=\"controller-server\">\n" + + " <binding>http://*/horizon/v1/*</binding>\n" + + " </handler>\n" + + + " <http>\n" + + " <server id='default' port='8080' />\n" + + " <filtering>\n" + + " <request-chain id='default'>\n" + + " <filter id='com.yahoo.vespa.hosted.controller.restapi.filter.ControllerAuthorizationFilter'/>\n" + + " <binding>http://*/*</binding>\n" + + " </request-chain>\n" + + " </filtering>\n" + + " </http>\n"; + } +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriterTest.java index ab9d50f8eae..d31d9c28c6c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriterTest.java @@ -22,33 +22,28 @@ public class TsdbQueryRewriterTest { @Test public void rewrites_query() throws IOException { - assertRewrite("filters-complex.json", "filters-complex.expected.json", Role.reader(TenantName.from("tenant2"))); + assertRewrite("filters-complex.json", "filters-complex.expected.json", Set.of(TenantName.from("tenant2")), false); assertRewrite("filter-in-execution-graph.json", "filter-in-execution-graph.expected.json", - Role.reader(TenantName.from("tenant2")), Role.athenzTenantAdmin(TenantName.from("tenant3"))); + Set.of(TenantName.from("tenant2"), TenantName.from("tenant3")), false); assertRewrite("filter-in-execution-graph.json", "filter-in-execution-graph.expected.operator.json", - Role.reader(TenantName.from("tenant2")), Role.athenzTenantAdmin(TenantName.from("tenant3")), Role.hostedOperator()); + Set.of(TenantName.from("tenant2"), TenantName.from("tenant3")), true); assertRewrite("no-filters.json", "no-filters.expected.json", - Role.reader(TenantName.from("tenant2")), Role.athenzTenantAdmin(TenantName.from("tenant3"))); + Set.of(TenantName.from("tenant2"), TenantName.from("tenant3")), false); assertRewrite("filters-meta-query.json", "filters-meta-query.expected.json", - Role.reader(TenantName.from("tenant2")), Role.athenzTenantAdmin(TenantName.from("tenant3"))); + Set.of(TenantName.from("tenant2"), TenantName.from("tenant3")), false); } - @Test(expected = TsdbQueryRewriter.UnauthorizedException.class) - public void throws_if_no_roles() throws IOException { - assertRewrite("filters-complex.json", "filters-complex.expected.json"); - } - - private static void assertRewrite(String initialFilename, String expectedFilename, Role... roles) throws IOException { + private static void assertRewrite(String initialFilename, String expectedFilename, Set<TenantName> tenants, boolean operator) throws IOException { byte[] data = Files.readAllBytes(Paths.get("src/test/resources/horizon", initialFilename)); - data = TsdbQueryRewriter.rewrite(data, Set.of(roles), SystemName.Public); + data = TsdbQueryRewriter.rewrite(data, tenants, operator, SystemName.Public); ByteArrayOutputStream baos = new ByteArrayOutputStream(); new JsonFormat(false).encode(baos, SlimeUtils.jsonToSlime(data)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json index ae3dc68d9e3..e883993cb53 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json @@ -14,17 +14,20 @@ "roles": [ "developer", "reader" - ] + ], + "enabled-horizon-dashboard":false }, "tenant1": { "roles": [ "administrator" - ] + ], + "enabled-horizon-dashboard":false }, "tenant2": { "roles": [ "developer" - ] + ], + "enabled-horizon-dashboard":false } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java index aa9775f1d43..1b86a0930d4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java @@ -5,7 +5,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 79b564eee52..3f805ba2916 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -24,7 +24,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.EndpointList; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index e3fef9f9066..a1108d5f03c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -12,7 +12,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; |