diff options
Diffstat (limited to 'controller-server/src')
13 files changed, 334 insertions, 284 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 6afeab9b4e6..c35e8c5a7ac 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -23,6 +23,7 @@ import com.yahoo.vespa.hosted.controller.config.ControllerConfig; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder; import com.yahoo.vespa.hosted.controller.notification.NotificationsDb; +import com.yahoo.vespa.hosted.controller.notify.Notifier; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.JobControlFlags; import com.yahoo.vespa.hosted.controller.security.AccessControl; @@ -88,6 +89,7 @@ public class Controller extends AbstractComponent { private final CuratorArchiveBucketDb archiveBucketDb; private final NotificationsDb notificationsDb; private final SupportAccessControl supportAccessControl; + private final Notifier notifier; /** * Creates a controller @@ -126,6 +128,7 @@ public class Controller extends AbstractComponent { auditLogger = new AuditLogger(curator, clock); jobControl = new JobControl(new JobControlFlags(curator, flagSource)); archiveBucketDb = new CuratorArchiveBucketDb(this); + notifier = new Notifier(curator, serviceRegistry.mailer()); notificationsDb = new NotificationsDb(this); supportAccessControl = new SupportAccessControl(this); @@ -330,4 +333,7 @@ public class Controller extends AbstractComponent { return supportAccessControl; } + public Notifier notifier() { + return notifier; + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java index 5c7ae5041fe..258884a4d11 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java @@ -4,6 +4,9 @@ package com.yahoo.vespa.hosted.controller.application.pkg; import com.google.common.hash.Funnel; import com.google.common.hash.Hashing; import com.yahoo.component.Version; +import com.yahoo.compress.ArchiveStreamReader; +import com.yahoo.compress.ArchiveStreamReader.ArchiveFile; +import com.yahoo.compress.ArchiveStreamReader.Options; import com.yahoo.config.application.FileSystemWrapper; import com.yahoo.config.application.FileSystemWrapper.FileWrapper; import com.yahoo.config.application.XmlPreProcessor; @@ -24,6 +27,7 @@ import com.yahoo.yolean.Exceptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.InputStreamReader; +import java.io.OutputStream; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; @@ -40,6 +44,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.SortedMap; +import java.util.TreeMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.function.Function; import java.util.function.Predicate; @@ -108,7 +113,7 @@ public class ApplicationPackage { this.trustedCertificates = files.get(trustedCertificatesFile).map(bytes -> X509CertificateUtils.certificateListFromPem(new String(bytes, UTF_8))).orElse(List.of()); - this.bundleHash = calculateBundleHash(); + this.bundleHash = calculateBundleHash(zippedContent); preProcessAndPopulateCache(); } @@ -120,7 +125,7 @@ public class ApplicationPackage { byte[] certificatesBytes = X509CertificateUtils.toPem(trustedCertificates).getBytes(UTF_8); ByteArrayOutputStream modified = new ByteArrayOutputStream(zippedContent.length + certificatesBytes.length); - ZipStreamReader.transferAndWrite(modified, new ByteArrayInputStream(zippedContent), trustedCertificatesFile, certificatesBytes); + ZipEntries.transferAndWrite(modified, new ByteArrayInputStream(zippedContent), trustedCertificatesFile, certificatesBytes); return new ApplicationPackage(modified.toByteArray()); } @@ -227,15 +232,23 @@ public class ApplicationPackage { } // Hashes all files and settings that require a deployment to be forwarded to configservers - private String calculateBundleHash() { + private String calculateBundleHash(byte[] zippedContent) { Predicate<String> entryMatcher = name -> ! name.endsWith(deploymentFile) && ! name.endsWith(buildMetaFile); - SortedMap<String, Long> entryCRCs = ZipStreamReader.getEntryCRCs(new ByteArrayInputStream(zippedContent), entryMatcher); - Funnel<SortedMap<String, Long>> funnel = (from, into) -> from.entrySet().forEach(entry -> { - into.putBytes(entry.getKey().getBytes()); - into.putLong(entry.getValue()); + SortedMap<String, Long> crcByEntry = new TreeMap<>(); + Options options = Options.standard().pathPredicate(entryMatcher); + ArchiveFile file; + try (ArchiveStreamReader reader = ArchiveStreamReader.ofZip(new ByteArrayInputStream(zippedContent), options)) { + OutputStream discard = OutputStream.nullOutputStream(); + while ((file = reader.readNextTo(discard)) != null) { + crcByEntry.put(file.path().toString(), file.crc32().orElse(-1)); + } + } + Funnel<SortedMap<String, Long>> funnel = (from, into) -> from.forEach((key, value) -> { + into.putBytes(key.getBytes()); + into.putLong(value); }); return Hashing.sha1().newHasher() - .putObject(entryCRCs, funnel) + .putObject(crcByEntry, funnel) .putInt(deploymentSpec.deployableHashCode()) .hash().toString(); } @@ -285,13 +298,13 @@ public class ApplicationPackage { } private Map<Path, Optional<byte[]>> read(Collection<String> names) { - var entries = new ZipStreamReader(new ByteArrayInputStream(zip), - name -> names.contains(withoutLegacyDir(name)), - maxSize, - true) - .entries().stream() - .collect(toMap(entry -> Paths.get(withoutLegacyDir(entry.zipEntry().getName())).normalize(), - ZipStreamReader.ZipEntryWithContent::content)); + var entries = ZipEntries.from(zip, + name -> names.contains(withoutLegacyDir(name)), + maxSize, + true) + .asList().stream() + .collect(toMap(entry -> Paths.get(withoutLegacyDir(entry.name())).normalize(), + ZipEntries.ZipEntryWithContent::content)); names.stream().map(Paths::get).forEach(path -> entries.putIfAbsent(path.normalize(), Optional.empty())); return entries; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiff.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiff.java index 97810b9de80..e18f6247cb1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiff.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiff.java @@ -15,7 +15,7 @@ import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.yahoo.vespa.hosted.controller.application.pkg.ZipStreamReader.ZipEntryWithContent; +import static com.yahoo.vespa.hosted.controller.application.pkg.ZipEntries.ZipEntryWithContent; /** * @author freva @@ -75,8 +75,8 @@ public class ApplicationPackageDiff { } private static Map<String, ZipEntryWithContent> readContents(ApplicationPackage app, int maxFileSizeToDiff) { - return new ZipStreamReader(new ByteArrayInputStream(app.zippedContent()), entry -> true, maxFileSizeToDiff, false).entries().stream() - .collect(Collectors.toMap(entry -> entry.zipEntry().getName(), e -> e)); + return ZipEntries.from(app.zippedContent(), entry -> true, maxFileSizeToDiff, false).asList().stream() + .collect(Collectors.toMap(ZipEntryWithContent::name, e -> e)); } private static List<String> lines(byte[] data) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java new file mode 100644 index 00000000000..a6cb7f23fc3 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java @@ -0,0 +1,99 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application.pkg; + +import com.yahoo.compress.ArchiveStreamReader; +import com.yahoo.compress.ArchiveStreamReader.ArchiveFile; +import com.yahoo.compress.ArchiveStreamReader.Options; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +/** + * A list of entries read from a ZIP archive, and their contents. + * + * @author bratseth + */ +public class ZipEntries { + + private final List<ZipEntryWithContent> entries; + + private ZipEntries(List<ZipEntryWithContent> entries) { + this.entries = List.copyOf(Objects.requireNonNull(entries)); + } + + /** Copies the zipped content from in to out, adding/overwriting an entry with the given name and content. */ + public static void transferAndWrite(OutputStream out, InputStream in, String name, byte[] content) { + try (ZipOutputStream zipOut = new ZipOutputStream(out); + ZipInputStream zipIn = new ZipInputStream(in)) { + for (ZipEntry entry = zipIn.getNextEntry(); entry != null; entry = zipIn.getNextEntry()) { + if (entry.getName().equals(name)) + continue; + + zipOut.putNextEntry(new ZipEntry(entry.getName())); + zipIn.transferTo(zipOut); + zipOut.closeEntry(); + } + zipOut.putNextEntry(new ZipEntry(name)); + zipOut.write(content); + zipOut.closeEntry(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** Read ZIP entries from inputStream */ + public static ZipEntries from(byte[] zip, Predicate<String> entryNameMatcher, int maxEntrySizeInBytes, boolean throwIfEntryExceedsMaxSize) { + Options options = Options.standard() + .pathPredicate(entryNameMatcher) + .maxSize(2 * (long) Math.pow(1024, 3)) // 2 GB + .maxEntrySize(maxEntrySizeInBytes) + .maxEntries(1024) + .truncateEntry(!throwIfEntryExceedsMaxSize); + List<ZipEntryWithContent> entries = new ArrayList<>(); + try (ArchiveStreamReader reader = ArchiveStreamReader.ofZip(new ByteArrayInputStream(zip), options)) { + ArchiveFile file; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + while ((file = reader.readNextTo(baos)) != null) { + entries.add(new ZipEntryWithContent(file.path().toString(), + Optional.of(baos.toByteArray()).filter(b -> b.length > 0), + file.size())); + baos.reset(); + } + } + return new ZipEntries(entries); + } + + public List<ZipEntryWithContent> asList() { return entries; } + + public static class ZipEntryWithContent { + + private final String name; + private final Optional<byte[]> content; + private final long size; + + public ZipEntryWithContent(String name, Optional<byte[]> content, long size) { + this.name = name; + this.content = content; + this.size = size; + } + + public String name() { return name; } + public byte[] contentOrThrow() { return content.orElseThrow(); } + public Optional<byte[]> content() { return content; } + public long size() { return size; } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java deleted file mode 100644 index 174ac4cb8b0..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application.pkg; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.UncheckedIOException; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.function.Predicate; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; -import java.util.zip.ZipOutputStream; - -/** - * @author bratseth - */ -public class ZipStreamReader { - - private final List<ZipEntryWithContent> entries = new ArrayList<>(); - private final int maxEntrySizeInBytes; - - public ZipStreamReader(InputStream input, Predicate<String> entryNameMatcher, int maxEntrySizeInBytes, boolean throwIfEntryExceedsMaxSize) { - this.maxEntrySizeInBytes = maxEntrySizeInBytes; - try (ZipInputStream zipInput = new ZipInputStream(input)) { - ZipEntry zipEntry; - - while (null != (zipEntry = zipInput.getNextEntry())) { - if (!entryNameMatcher.test(requireName(zipEntry.getName()))) continue; - entries.add(readContent(zipEntry, zipInput, throwIfEntryExceedsMaxSize)); - } - } catch (IOException e) { - throw new UncheckedIOException("IO error reading zip content", e); - } - } - - /** Copies the zipped content from in to out, adding/overwriting an entry with the given name and content. */ - public static void transferAndWrite(OutputStream out, InputStream in, String name, byte[] content) { - try (ZipOutputStream zipOut = new ZipOutputStream(out); - ZipInputStream zipIn = new ZipInputStream(in)) { - for (ZipEntry entry = zipIn.getNextEntry(); entry != null; entry = zipIn.getNextEntry()) { - if (entry.getName().equals(name)) - continue; - - zipOut.putNextEntry(new ZipEntry(entry.getName())); - zipIn.transferTo(zipOut); - zipOut.closeEntry(); - } - zipOut.putNextEntry(new ZipEntry(name)); - zipOut.write(content); - zipOut.closeEntry(); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public static SortedMap<String, Long> getEntryCRCs(InputStream in, Predicate<String> entryNameMatcher) { - SortedMap<String, Long> entryCRCs = new TreeMap<>(); - byte[] buffer = new byte[2048]; - try (ZipInputStream zipIn = new ZipInputStream(in)) { - for (ZipEntry entry = zipIn.getNextEntry(); entry != null; entry = zipIn.getNextEntry()) { - if ( ! entryNameMatcher.test(entry.getName())) - continue; - // CRC is not set until entry is read - while ( -1 != zipIn.read(buffer)){} - entryCRCs.put(entry.getName(), entry.getCrc()); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - return entryCRCs; - } - - private ZipEntryWithContent readContent(ZipEntry zipEntry, ZipInputStream zipInput, boolean throwIfEntryExceedsMaxSize) { - try (ByteArrayOutputStream bis = new ByteArrayOutputStream()) { - byte[] buffer = new byte[2048]; - int read; - long size = 0; - while ( -1 != (read = zipInput.read(buffer))) { - size += read; - if (size > maxEntrySizeInBytes) { - if (throwIfEntryExceedsMaxSize) throw new IllegalArgumentException( - "Entry in zip content exceeded size limit of " + maxEntrySizeInBytes + " bytes"); - } else bis.write(buffer, 0, read); - } - - boolean hasContent = size <= maxEntrySizeInBytes; - return new ZipEntryWithContent(zipEntry, - Optional.of(bis).filter(__ -> hasContent).map(ByteArrayOutputStream::toByteArray), - size); - } catch (IOException e) { - throw new UncheckedIOException("Failed reading from zipped content", e); - } - } - - public List<ZipEntryWithContent> entries() { return Collections.unmodifiableList(entries); } - - private static String requireName(String name) { - if (List.of(name.split("/")).contains("..") || - !trimTrailingSlash(name).equals(Path.of(name).normalize().toString())) { - throw new IllegalArgumentException("Unexpected non-normalized path found in zip content: '" + name + "'"); - } - return name; - } - - private static String trimTrailingSlash(String name) { - if (name.endsWith("/")) return name.substring(0, name.length() - 1); - return name; - } - - public static class ZipEntryWithContent { - - private final ZipEntry zipEntry; - private final Optional<byte[]> content; - private final long size; - - public ZipEntryWithContent(ZipEntry zipEntry, Optional<byte[]> content, long size) { - this.zipEntry = zipEntry; - this.content = content; - this.size = size; - } - - public ZipEntry zipEntry() { return zipEntry; } - public byte[] contentOrThrow() { return content.orElseThrow(); } - public Optional<byte[]> content() { return content; } - public long size() { return size; } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java index c0bd1ac03ff..5244d46d0a9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.notify.Notifier; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.time.Clock; @@ -32,14 +33,16 @@ public class NotificationsDb { private final Clock clock; private final CuratorDb curatorDb; + private final Notifier notifier; public NotificationsDb(Controller controller) { - this(controller.clock(), controller.curator()); + this(controller.clock(), controller.curator(), controller.notifier()); } - NotificationsDb(Clock clock, CuratorDb curatorDb) { + NotificationsDb(Clock clock, CuratorDb curatorDb, Notifier notifier) { this.clock = clock; this.curatorDb = curatorDb; + this.notifier = notifier; } public List<TenantName> listTenantsWithNotifications() { @@ -61,13 +64,24 @@ public class NotificationsDb { * already exists, it'll be replaced by this one instead */ public void setNotification(NotificationSource source, Type type, Level level, List<String> messages) { + Optional<Notification> changed = Optional.empty(); try (Lock lock = curatorDb.lockNotifications(source.tenant())) { - List<Notification> notifications = curatorDb.readNotifications(source.tenant()).stream() + var existingNotifications = curatorDb.readNotifications(source.tenant()); + List<Notification> notifications = existingNotifications.stream() .filter(notification -> !source.equals(notification.source()) || type != notification.type()) .collect(Collectors.toCollection(ArrayList::new)); - notifications.add(new Notification(clock.instant(), type, level, source, messages)); + var notification = new Notification(clock.instant(), type, level, source, messages); + // Be conservative for now, only dispatch notifications if they are from new source or with new type. + // the message content and level is ignored for now + if (!existingNotifications.stream().anyMatch(n -> n.source().equals(source) && n.type().equals(type))) { + changed = Optional.of(notification); + } + notifications.add(notification); curatorDb.writeNotifications(source.tenant(), notifications); } + if (changed.isPresent()) { + notifier.dispatch(changed.get()); + } } /** Remove the notification with the given source and type */ @@ -131,8 +145,9 @@ public class NotificationsDb { newNotifications.stream()) .collect(Collectors.toUnmodifiableList()); - if (!initial.equals(updated)) + if (!initial.equals(updated)) { curatorDb.writeNotifications(deploymentSource.tenant(), updated); + } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notify/Notifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notify/Notifier.java new file mode 100644 index 00000000000..46e1fd904ed --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notify/Notifier.java @@ -0,0 +1,76 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.notify; + +import com.yahoo.text.Text; +import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; +import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; +import com.yahoo.vespa.hosted.controller.api.integration.organization.MailerException; +import com.yahoo.vespa.hosted.controller.notification.Notification; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; + +import java.util.Collection; +import java.util.Objects; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Notifier is responsible for dispatching user notifications to their chosen Contact points. + * + * @author enygaard + */ +public class Notifier { + private final CuratorDb curatorDb; + private final Mailer mailer; + + private static final Logger log = Logger.getLogger(Notifier.class.getName()); + + public Notifier(CuratorDb curatorDb, Mailer mailer) { + this.curatorDb = Objects.requireNonNull(curatorDb); + this.mailer = Objects.requireNonNull(mailer); + } + + public void dispatch(Notification notification) { + var tenant = curatorDb.readTenant(notification.source().tenant()); + tenant.stream().forEach(t -> { + if (t instanceof CloudTenant) { + var ct = (CloudTenant) t; + ct.info().contacts().all().stream() + .filter(c -> c.audiences().contains(TenantContacts.Audience.NOTIFICATIONS)) + .collect(Collectors.groupingBy(TenantContacts.Contact::type, Collectors.toList())) + .entrySet() + .forEach(e -> dispatch(notification, e.getKey(), e.getValue())); + } + }); + } + + private void dispatch(Notification notification, TenantContacts.Type type, Collection<? extends TenantContacts.Contact> contacts) { + switch (type) { + case EMAIL: + dispatch(notification, contacts.stream().map(c -> (TenantContacts.EmailContact) c).collect(Collectors.toList())); + break; + default: + throw new IllegalArgumentException("Unknown TenantContacts type " + type.name()); + } + } + + private void dispatch(Notification notification, Collection<TenantContacts.EmailContact> contacts) { + try { + mailer.send(mailOf(notification, contacts.stream().map(c -> c.email()).collect(Collectors.toList()))); + } catch (MailerException e) { + log.log(Level.SEVERE, "Failed sending email", e); + } + } + + private Mail mailOf(Notification n, Collection<String> recipients) { + var subject = Text.format("[%s] Vespa Notification for %s", n.level().toString().toUpperCase(), n.type().name()); + var body = new StringBuilder(); + body.append("Source: ").append(n.source().toString()).append("\n") + .append("\n") + .append(String.join("\n", n.messages())); + return new Mail(recipients, subject.toString(), body.toString()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java index 766a51e3e8d..21e803800f5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java @@ -104,7 +104,7 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { Optional<ApplicationName> application = Optional.ofNullable(path.get("application")).map(ApplicationName::from); final Optional<ZoneId> zone; - if(path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/{*}")) { + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/{*}")) { zone = Optional.of(ZoneId.from(path.get("environment"), path.get("region"))); } else if(path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/{*}")) { zone = Optional.of(ZoneId.from(path.get("environment"), path.get("region"))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java index 709a7967a5e..4e155e937b9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java @@ -5,7 +5,6 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import org.junit.Test; -import java.io.ByteArrayInputStream; import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; @@ -154,9 +153,9 @@ public class ApplicationPackageTest { } private static Map<String, String> unzip(byte[] zip) { - return new ZipStreamReader(new ByteArrayInputStream(zip), __ -> true, 1 << 10, true) - .entries().stream() - .collect(Collectors.toMap(entry -> entry.zipEntry().getName(), + return ZipEntries.from(zip, __ -> true, 1 << 10, true) + .asList().stream() + .collect(Collectors.toMap(ZipEntries.ZipEntryWithContent::name, entry -> new String(entry.contentOrThrow(), UTF_8))); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntriesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntriesTest.java new file mode 100644 index 00000000000..6908464640b --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntriesTest.java @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application.pkg; + +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; +import org.junit.Test; + +import javax.security.auth.x500.X500Principal; +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class ZipEntriesTest { + + @Test + public void test_replacement() { + ApplicationPackage applicationPackage = new ApplicationPackage(new byte[0]); + List<X509Certificate> certificates = IntStream.range(0, 3) + .mapToObj(i -> { + KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + X500Principal subject = new X500Principal("CN=subject" + i); + return X509CertificateBuilder.fromKeypair(keyPair, + subject, + Instant.now(), + Instant.now().plusSeconds(1), + SignatureAlgorithm.SHA512_WITH_ECDSA, + BigInteger.valueOf(1)) + .build(); + }) + .collect(Collectors.toUnmodifiableList()); + + assertEquals(List.of(), applicationPackage.trustedCertificates()); + for (int i = 0; i < certificates.size(); i++) { + applicationPackage = applicationPackage.withTrustedCertificate(certificates.get(i)); + assertEquals(certificates.subList(0, i + 1), applicationPackage.trustedCertificates()); + } + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReaderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReaderTest.java deleted file mode 100644 index 33c18d123d2..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReaderTest.java +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application.pkg; - -import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.SignatureAlgorithm; -import com.yahoo.security.X509CertificateBuilder; -import org.junit.Test; - -import javax.security.auth.x500.X500Principal; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.math.BigInteger; -import java.nio.charset.StandardCharsets; -import java.security.KeyPair; -import java.security.cert.X509Certificate; -import java.time.Instant; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * @author mpolden - */ -public class ZipStreamReaderTest { - - @Test - public void test_size_limit() { - Map<String, String> entries = Map.of("foo.xml", "foobar"); - try { - new ZipStreamReader(new ByteArrayInputStream(zip(entries)), "foo.xml"::equals, 1, true); - fail("Expected exception"); - } catch (IllegalArgumentException ignored) {} - - entries = Map.of("foo.xml", "foobar", - "foo.jar", "0".repeat(100) // File not extracted and thus not subject to size limit - ); - ZipStreamReader reader = new ZipStreamReader(new ByteArrayInputStream(zip(entries)), "foo.xml"::equals, 10, true); - byte[] extracted = reader.entries().get(0).contentOrThrow(); - assertEquals("foobar", new String(extracted, StandardCharsets.UTF_8)); - } - - @Test - public void test_paths() { - Map<String, Boolean> tests = Map.of( - "../../services.xml", true, - "/../.././services.xml", true, - "./application/././services.xml", true, - "application//services.xml", true, - "artifacts/", false, // empty dir - "services..xml", false, - "application/services.xml", false, - "components/foo-bar-deploy.jar", false, - "services.xml", false - ); - tests.forEach((name, expectException) -> { - try { - new ZipStreamReader(new ByteArrayInputStream(zip(Map.of(name, "foo"))), name::equals, 1024, true); - assertFalse("Expected exception for '" + name + "'", expectException); - } catch (IllegalArgumentException ignored) { - assertTrue("Unexpected exception for '" + name + "'", expectException); - } - }); - } - - @Test - public void test_replacement() { - ApplicationPackage applicationPackage = new ApplicationPackage(new byte[0]); - List<X509Certificate> certificates = IntStream.range(0, 3) - .mapToObj(i -> { - KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); - X500Principal subject = new X500Principal("CN=subject" + i); - return X509CertificateBuilder.fromKeypair(keyPair, - subject, - Instant.now(), - Instant.now().plusSeconds(1), - SignatureAlgorithm.SHA512_WITH_ECDSA, - BigInteger.valueOf(1)) - .build(); - }) - .collect(Collectors.toUnmodifiableList()); - - assertEquals(List.of(), applicationPackage.trustedCertificates()); - for (int i = 0; i < certificates.size(); i++) { - applicationPackage = applicationPackage.withTrustedCertificate(certificates.get(i)); - assertEquals(certificates.subList(0, i + 1), applicationPackage.trustedCertificates()); - } - } - - private static byte[] zip(Map<String, String> entries) { - ByteArrayOutputStream zip = new ByteArrayOutputStream(); - try (ZipOutputStream out = new ZipOutputStream(zip)) { - for (Map.Entry<String, String> entry : entries.entrySet()) { - out.putNextEntry(new ZipEntry(entry.getKey())); - out.write(entry.getValue().getBytes(StandardCharsets.UTF_8)); - out.closeEntry(); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - return zip.toByteArray(); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 989a7c31821..86c21839c96 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -32,7 +32,6 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.application.pkg.ZipStreamReader; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher; @@ -41,8 +40,6 @@ import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; import javax.security.auth.x500.X500Principal; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.math.BigInteger; import java.security.KeyPair; import java.security.cert.X509Certificate; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java index d51856b329d..a5655d2e6eb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.notification; +import com.google.common.collect.ImmutableBiMap; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.TenantName; @@ -11,8 +12,14 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.notify.Notifier; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; +import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; +import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; import org.junit.Before; import org.junit.Test; @@ -22,6 +29,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.controller.notification.Notification.Level; @@ -36,6 +44,19 @@ import static org.junit.Assert.assertTrue; public class NotificationsDbTest { private static final TenantName tenant = TenantName.from("tenant1"); + private static final String email = "user1@example.com"; + private static final CloudTenant cloudTenant = new CloudTenant(tenant, + Instant.now(), + LastLoginInfo.EMPTY, + Optional.empty(), + ImmutableBiMap.of(), + TenantInfo.empty() + .withContacts(new TenantContacts( + List.of(new TenantContacts.EmailContact( + List.of(TenantContacts.Audience.NOTIFICATIONS), + email)))), + List.of(), + Optional.empty()); private static final List<Notification> notifications = List.of( notification(1001, Type.deployment, Level.error, NotificationSource.from(tenant), "tenant msg"), notification(1101, Type.applicationPackage, Level.warning, NotificationSource.from(TenantAndApplicationId.from(tenant.value(), "app1")), "app msg"), @@ -46,7 +67,8 @@ public class NotificationsDbTest { private final ManualClock clock = new ManualClock(Instant.ofEpochSecond(12345)); private final MockCuratorDb curatorDb = new MockCuratorDb(); - private final NotificationsDb notificationsDb = new NotificationsDb(clock, curatorDb); + private final MockMailer mailer = new MockMailer(); + private final NotificationsDb notificationsDb = new NotificationsDb(clock, curatorDb, new Notifier(curatorDb, mailer)); @Test public void list_test() { @@ -75,6 +97,29 @@ public class NotificationsDbTest { } @Test + public void notifier_test() { + Notification notification1 = notification(12345, Type.deployment, Level.warning, NotificationSource.from(ApplicationId.from(tenant.value(), "app2", "instance2")), "instance msg #2"); + Notification notification2 = notification(12345, Type.deployment, Level.error, NotificationSource.from(ApplicationId.from(tenant.value(), "app3", "instance2")), "instance msg #3"); + Notification notification3 = notification(12345, Type.reindex, Level.warning, NotificationSource.from(ApplicationId.from(tenant.value(), "app2", "instance2")), "instance msg #2"); + + var a = notifications.get(0); + notificationsDb.setNotification(a.source(), a.type(), a.level(), a.messages()); + assertEquals(0, mailer.inbox(email).size()); + + // Replace the 3rd notification. but don't change source or type + notificationsDb.setNotification(notification1.source(), notification1.type(), notification1.level(), notification1.messages()); + assertEquals(0, mailer.inbox(email).size()); + + // Notification for a new app, add without replacement + notificationsDb.setNotification(notification2.source(), notification2.type(), notification2.level(), notification2.messages()); + assertEquals(1, mailer.inbox(email).size()); + + // Notification for new type on existing app + notificationsDb.setNotification(notification3.source(), notification3.type(), notification3.level(), notification3.messages()); + assertEquals(2, mailer.inbox(email).size()); + } + + @Test public void remove_single_test() { // Remove the 3rd notification notificationsDb.removeNotification(NotificationSource.from(ApplicationId.from(tenant.value(), "app2", "instance2")), Type.deployment); @@ -160,6 +205,8 @@ public class NotificationsDbTest { @Before public void init() { curatorDb.writeNotifications(tenant, notifications); + curatorDb.writeTenant(cloudTenant); + mailer.reset(); } private static List<Notification> notificationIndices(int... indices) { |