Add custom email content to notification

Render emails using Apache Velocity

11 files changed, 378 insertions, 233 deletions

diff --git a/controller-server/pom.xml b/controller-server/pom.xml
index 6671b71c73f..a9db2cace85 100644
--- a/controller-server/pom.xml
+++ b/controller-server/pom.xml
@@ -118,6 +118,33 @@
         <!-- compile -->
 
         <dependency>
+            <groupId>org.apache.velocity</groupId>
+            <artifactId>velocity-engine-core</artifactId>
+            <exclusions>
+                <exclusion>
+                    <!-- Must use the one provided by Jdisc to prevent two instances of slf4j classes. -->
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-api</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.velocity.tools</groupId>
+            <artifactId>velocity-tools-generic</artifactId>
+            <exclusions>
+                <exclusion>
+                    <!-- Must use the one provided by Jdisc to prevent two instances of slf4j classes. -->
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>commons-logging</groupId>
+                    <artifactId>commons-logging</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-csv</artifactId>
         </dependency>
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java
index d22efdc5f6e..48e9d1f6786 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java
@@ -2,8 +2,11 @@
 package com.yahoo.vespa.hosted.controller.notification;
 
 import java.time.Instant;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * Represents an event that we want to notify the tenant about. The message(s) should be short
@@ -13,15 +16,21 @@ import java.util.Objects;
  *
  * @author freva
  */
-public record Notification(Instant at, com.yahoo.vespa.hosted.controller.notification.Notification.Type type, com.yahoo.vespa.hosted.controller.notification.Notification.Level level, NotificationSource source, List<String> messages) {
+public record Notification(Instant at, Notification.Type type, Notification.Level level, NotificationSource source,
+                           List<String> messages, Optional<MailContent> mailContent) {
 
     public Notification(Instant at, Type type, Level level, NotificationSource source, List<String> messages) {
-        this.at = Objects.requireNonNull(at, "at cannot be null");
-        this.type = Objects.requireNonNull(type, "type cannot be null");
-        this.level = Objects.requireNonNull(level, "level cannot be null");
-        this.source = Objects.requireNonNull(source, "source cannot be null");
-        this.messages = List.copyOf(Objects.requireNonNull(messages, "messages cannot be null"));
+        this(at, type, level, source, messages, Optional.empty());
+    }
+
+    public Notification {
+        at = Objects.requireNonNull(at, "at cannot be null");
+        type = Objects.requireNonNull(type, "type cannot be null");
+        level = Objects.requireNonNull(level, "level cannot be null");
+        source = Objects.requireNonNull(source, "source cannot be null");
+        messages = List.copyOf(Objects.requireNonNull(messages, "messages cannot be null"));
         if (messages.size() < 1) throw new IllegalArgumentException("messages cannot be empty");
+        mailContent = Objects.requireNonNull(mailContent);
     }
 
     public enum Level {
@@ -63,4 +72,36 @@ public record Notification(Instant at, com.yahoo.vespa.hosted.controller.notific
 
     }
 
+    public static class MailContent {
+        private final String template;
+        private final Map<String, Object> values;
+        private final String subject;
+
+        private MailContent(Builder b) {
+            template = Objects.requireNonNull(b.template);
+            values = Map.copyOf(b.values);
+            subject = b.subject;
+        }
+
+        public String template() { return template; }
+        public Map<String, Object> values() { return Map.copyOf(values); }
+        public Optional<String> subject() { return Optional.ofNullable(subject); }
+
+        public static Builder fromTemplate(String template) { return new Builder(template); }
+
+        public static class Builder {
+            private final String template;
+            private final HashMap<String, Object> values = new HashMap<>();
+            private String subject;
+
+            private Builder(String template) {
+                this.template = template;
+            }
+
+            public Builder with(String name, Object value) { values.put(name, value); return this; }
+            public Builder subject(String s) { this.subject = s; return this; }
+            public MailContent build() { return new MailContent(this); }
+        }
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java
index afb260bf765..c1e1f075552 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java
@@ -16,8 +16,17 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 import com.yahoo.vespa.hosted.controller.tenant.CloudTenant;
 import com.yahoo.vespa.hosted.controller.tenant.TenantContacts;
-
+import com.yahoo.yolean.Exceptions;
+import org.apache.velocity.VelocityContext;
+import org.apache.velocity.app.Velocity;
+import org.apache.velocity.app.VelocityEngine;
+import org.apache.velocity.runtime.resource.loader.StringResourceLoader;
+import org.apache.velocity.runtime.resource.util.StringResourceRepository;
+import org.apache.velocity.tools.generic.EscapeTool;
+
+import java.io.StringWriter;
 import java.net.URI;
+import java.nio.charset.StandardCharsets;
 import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
@@ -26,8 +35,6 @@ import java.util.logging.Logger;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
-import static com.yahoo.yolean.Exceptions.uncheck;
-
 /**
  * Notifier is responsible for dispatching user notifications to their chosen Contact points.
  *
@@ -39,6 +46,7 @@ public class Notifier {
     private final FlagSource flagSource;
     private final NotificationFormatter formatter;
     private final URI dashboardUri;
+    private final VelocityEngine velocity;
 
     private static final Logger log = Logger.getLogger(Notifier.class.getName());
 
@@ -51,6 +59,28 @@ public class Notifier {
         this.flagSource = Objects.requireNonNull(flagSource);
         this.formatter = new NotificationFormatter(zoneRegistry);
         this.dashboardUri = zoneRegistry.dashboardUrl();
+        this.velocity = createTemplateEngine();
+    }
+
+    private static VelocityEngine createTemplateEngine() {
+        var v = new VelocityEngine();
+        v.setProperty(Velocity.RESOURCE_LOADERS, "string");
+        v.setProperty(Velocity.RESOURCE_LOADER + ".string.class", StringResourceLoader.class.getName());
+        v.setProperty(Velocity.RESOURCE_LOADER + ".string.repository.static", "false");
+        v.init();
+        var repo = (StringResourceRepository) v.getApplicationAttribute(StringResourceLoader.REPOSITORY_NAME_DEFAULT);
+        registerTemplate(repo, "mail");
+        registerTemplate(repo, "default-mail-content");
+        registerTemplate(repo, "notification-message");
+        return v;
+    }
+
+    private static void registerTemplate(StringResourceRepository repo, String name) {
+        var templateStr = Exceptions.uncheck(() -> {
+            var in = Notifier.class.getResourceAsStream("/mail/%s.vm".formatted(name));
+            return new String(in.readAllBytes());
+        });
+        repo.putStringResource(name, templateStr);
     }
 
     public void dispatch(List<Notification> notifications, NotificationSource source) {
@@ -114,23 +144,43 @@ public class Notifier {
 
     public Mail mailOf(FormattedNotification content, Collection<String> recipients) {
         var notification = content.notification();
-        var subject = Text.format("[%s] %s Vespa Notification for %s", notification.level().toString().toUpperCase(), content.prettyType(), applicationIdSource(notification.source()));
-        var template = uncheck(() -> Notifier.class.getResourceAsStream("/mail/mail-notification.tmpl").readAllBytes());
-        var html = new String(template)
-                .replace("[[NOTIFICATION_HEADER]]", content.messagePrefix())
-                .replace("[[NOTIFICATION_ITEMS]]", notification.messages().stream()
-                        .map(Notifier::linkify)
-                        .map(Notifier::capitalise)
-                        .map(m -> "<p>" + m + "</p>")
-                        .collect(Collectors.joining()))
-                .replace("[[LINK_TO_NOTIFICATION]]", notificationLink(notification.source()))
-                .replace("[[LINK_TO_ACCOUNT_NOTIFICATIONS]]", accountNotificationsUri(content.notification().source().tenant()))
-                .replace("[[LINK_TO_PRIVACY_POLICY]]", "https://legal.yahoo.com/xw/en/yahoo/privacy/topic/b2bprivacypolicy/index.html")
-                .replace("[[LINK_TO_TERMS_OF_SERVICE]]", consoleUri("terms-of-service-trial.html"))
-                .replace("[[LINK_TO_SUPPORT]]", consoleUri("support"));
+        var subject = content.notification().mailContent().flatMap(Notification.MailContent::subject)
+                .orElseGet(() -> Text.format(
+                        "[%s] %s Vespa Notification for %s", notification.level().toString().toUpperCase(),
+                        content.prettyType(), applicationIdSource(notification.source())));
+        var html = generateHtml(content);
         return new Mail(recipients, subject, "", html);
     }
 
+    private String generateHtml(FormattedNotification content) {
+        var esc = new EscapeTool();
+        var mailContent = content.notification().mailContent().orElseGet(() -> generateContentFromMessages(content, esc));
+        var ctx = new VelocityContext();
+        ctx.put("esc", esc);
+        ctx.put("accountNotificationLink", accountNotificationsUri(content.notification().source().tenant()));
+        ctx.put("privacyPolicyLink", "https://legal.yahoo.com/xw/en/yahoo/privacy/topic/b2bprivacypolicy/index.html");
+        ctx.put("termsOfServiceLink", consoleUri("terms-of-service-trial.html"));
+        ctx.put("supportLink", consoleUri("support"));
+        ctx.put("mailBodyTemplate", mailContent.template());
+        mailContent.values().forEach(ctx::put);
+
+        var writer = new StringWriter();
+        // Ignoring return value - implementation either returns 'true' or throws, never 'false'
+        velocity.mergeTemplate("mail", StandardCharsets.UTF_8.name(), ctx, writer);
+        return writer.toString();
+    }
+
+    private Notification.MailContent generateContentFromMessages(FormattedNotification f, EscapeTool esc) {
+        var items = f.notification().messages().stream().map(m -> capitalise(linkify(esc.html(m)))).toList();
+        return Notification.MailContent.fromTemplate("default-mail-content")
+                .with("mailMessageTemplate", "notification-message")
+                .with("mailTitle", "Vespa Cloud Notifications")
+                .with("notificationHeader", f.messagePrefix())
+                .with("notificationItems", items)
+                .with("consoleLink", notificationLink(f.notification().source()))
+                .build();
+    }
+
     @VisibleForTesting
     static String linkify(String text) {
         return urlPattern.matcher(text).replaceAll((res) -> String.format("<a href=\"%s\">%s</a>", res.group(), res.group()));
diff --git a/controller-server/src/main/resources/mail/default-mail-content.vm b/controller-server/src/main/resources/mail/default-mail-content.vm
new file mode 100644
index 00000000000..02de98b900d
--- /dev/null
+++ b/controller-server/src/main/resources/mail/default-mail-content.vm
@@ -0,0 +1,131 @@
+<tbody>
+<tr>
+    <td
+            align="left"
+            style="
+                            font-size: 0px;
+                            padding: 0px 25px 0px 25px;
+                            padding-top: 0px;
+                            padding-right: 50px;
+                            padding-bottom: 0px;
+                            padding-left: 50px;
+                            word-break: break-word;
+                          "
+    >
+        <div
+                style="
+                              font-family: Open Sans, Helvetica, Arial,
+                                sans-serif;
+                              font-size: 13px;
+                              line-height: 22px;
+                              text-align: left;
+                              color: #797e82;
+                            "
+        >
+            <h1
+                    style="
+                                text-align: center;
+                                color: #000000;
+                                line-height: 32px;
+                              "
+            >
+                $esc.html($mailTitle)
+            </h1>
+        </div>
+    </td>
+</tr>
+<tr>
+    <td
+            align="left"
+            style="
+                            font-size: 0px;
+                            padding: 0px 25px 0px 25px;
+                            padding-top: 0px;
+                            padding-right: 50px;
+                            padding-bottom: 0px;
+                            padding-left: 50px;
+                            word-break: break-word;
+                          "
+    >
+        <div
+                style="
+                              font-family: Open Sans, Helvetica, Arial,
+                                sans-serif;
+                              font-size: 13px;
+                              line-height: 22px;
+                              text-align: left;
+                              color: #797e82;
+                            "
+        >
+
+            #parse($mailMessageTemplate)
+
+        </div>
+    </td>
+</tr>
+<tr>
+    <td
+            align="center"
+            vertical-align="middle"
+            style="
+                            font-size: 0px;
+                            padding: 10px 25px;
+                            padding-top: 20px;
+                            padding-bottom: 20px;
+                            word-break: break-word;
+                          "
+    >
+        <table
+                border="0"
+                cellpadding="0"
+                cellspacing="0"
+                role="presentation"
+                style="border-collapse: separate; line-height: 100%"
+        >
+            <tbody>
+            <tr>
+                <td
+                        align="center"
+                        bgcolor="#005A8E"
+                        role="presentation"
+                        style="
+                                    border: none;
+                                    border-radius: 100px;
+                                    cursor: auto;
+                                    mso-padding-alt: 15px 25px 15px 25px;
+                                    background: #005a8e;
+                                  "
+                        valign="middle"
+                >
+                    <a
+                            href="$consoleLink"
+                            style="
+                                      display: inline-block;
+                                      background: #005a8e;
+                                      color: #ffffff;
+                                      font-family: Open Sans, Helvetica, Arial,
+                                        sans-serif;
+                                      font-size: 13px;
+                                      font-weight: normal;
+                                      line-height: 120%;
+                                      margin: 0;
+                                      text-decoration: none;
+                                      text-transform: none;
+                                      padding: 15px 25px 15px 25px;
+                                      mso-padding-alt: 0px;
+                                      border-radius: 100px;
+                                    "
+                            target="_blank"
+                    ><b style="font-weight: 700"
+                    ><b style="font-weight: 700"
+                    >Go to Console</b
+                    ></b
+                    ></a
+                    >
+                </td>
+            </tr>
+            </tbody>
+        </table>
+    </td>
+</tr>
+</tbody>
\ No newline at end of file
diff --git a/controller-server/src/main/resources/mail/mail-notification.tmpl b/controller-server/src/main/resources/mail/mail.vm
index 5bf5530b433..1dbec781b3a 100644
--- a/controller-server/src/main/resources/mail/mail-notification.tmpl
+++ b/controller-server/src/main/resources/mail/mail.vm
@@ -383,138 +383,9 @@
                     style="vertical-align: top"
                     width="100%"
                   >
-                    <tbody>
-                      <tr>
-                        <td
-                          align="left"
-                          style="
-                            font-size: 0px;
-                            padding: 0px 25px 0px 25px;
-                            padding-top: 0px;
-                            padding-right: 50px;
-                            padding-bottom: 0px;
-                            padding-left: 50px;
-                            word-break: break-word;
-                          "
-                        >
-                          <div
-                            style="
-                              font-family: Open Sans, Helvetica, Arial,
-                                sans-serif;
-                              font-size: 13px;
-                              line-height: 22px;
-                              text-align: left;
-                              color: #797e82;
-                            "
-                          >
-                            <h1
-                              style="
-                                text-align: center;
-                                color: #000000;
-                                line-height: 32px;
-                              "
-                            >
-                              Vespa Cloud Notifications
-                            </h1>
-                          </div>
-                        </td>
-                      </tr>
-                      <tr>
-                        <td
-                          align="left"
-                          style="
-                            font-size: 0px;
-                            padding: 0px 25px 0px 25px;
-                            padding-top: 0px;
-                            padding-right: 50px;
-                            padding-bottom: 0px;
-                            padding-left: 50px;
-                            word-break: break-word;
-                          "
-                        >
-                          <div
-                            style="
-                              font-family: Open Sans, Helvetica, Arial,
-                                sans-serif;
-                              font-size: 13px;
-                              line-height: 22px;
-                              text-align: left;
-                              color: #797e82;
-                            "
-                          >
-                            <p>
-                              [[NOTIFICATION_HEADER]]:
-                            </p>
-                            [[NOTIFICATION_ITEMS]]
-                          </div>
-                        </td>
-                      </tr>
-                      <tr>
-                        <td
-                          align="center"
-                          vertical-align="middle"
-                          style="
-                            font-size: 0px;
-                            padding: 10px 25px;
-                            padding-top: 20px;
-                            padding-bottom: 20px;
-                            word-break: break-word;
-                          "
-                        >
-                          <table
-                            border="0"
-                            cellpadding="0"
-                            cellspacing="0"
-                            role="presentation"
-                            style="border-collapse: separate; line-height: 100%"
-                          >
-                            <tbody>
-                              <tr>
-                                <td
-                                  align="center"
-                                  bgcolor="#005A8E"
-                                  role="presentation"
-                                  style="
-                                    border: none;
-                                    border-radius: 100px;
-                                    cursor: auto;
-                                    mso-padding-alt: 15px 25px 15px 25px;
-                                    background: #005a8e;
-                                  "
-                                  valign="middle"
-                                >
-                                  <a
-                                    href="[[LINK_TO_NOTIFICATION]]"
-                                    style="
-                                      display: inline-block;
-                                      background: #005a8e;
-                                      color: #ffffff;
-                                      font-family: Open Sans, Helvetica, Arial,
-                                        sans-serif;
-                                      font-size: 13px;
-                                      font-weight: normal;
-                                      line-height: 120%;
-                                      margin: 0;
-                                      text-decoration: none;
-                                      text-transform: none;
-                                      padding: 15px 25px 15px 25px;
-                                      mso-padding-alt: 0px;
-                                      border-radius: 100px;
-                                    "
-                                    target="_blank"
-                                    ><b style="font-weight: 700"
-                                      ><b style="font-weight: 700"
-                                        >Go to Console</b
-                                      ></b
-                                    ></a
-                                  >
-                                </td>
-                              </tr>
-                            </tbody>
-                          </table>
-                        </td>
-                      </tr>
-                    </tbody>
+
+                  #parse($mailBodyTemplate)
+
                   </table>
                 </div>
                 <!--[if mso | IE]></td></tr></table><![endif]-->
@@ -592,7 +463,7 @@
                                 target="_blank"
                                 rel="noopener noreferrer"
                                 style="color: #005a8e"
-                                href="[[LINK_TO_PRIVACY_POLICY]]"
+                                href="$privacyPolicyLink"
                                 ><span style="color: #005a8e"
                                   >Yahoo Privacy Policy</span
                                 ></a
@@ -602,7 +473,7 @@
                                 target="_blank"
                                 rel="noopener noreferrer"
                                 style="color: #005a8e"
-                                href="[[LINK_TO_TERMS_OF_SERVICE]]"
+                                href="$termsOfServiceLink"
                                 ><span style="color: #005a8e"
                                   >Terms of Service</span
                                 ></a
@@ -612,7 +483,7 @@
                                 target="_blank"
                                 rel="noopener noreferrer"
                                 style="color: #005a8e"
-                                href="[[LINK_TO_SUPPORT]]"
+                                href="$supportLink"
                                 ><span style="color: #005a8e">Support</span></a
                               >
                             </p>
@@ -621,7 +492,7 @@
                                 target="_blank"
                                 rel="noopener noreferrer"
                                 style="color: inherit; text-decoration: none"
-                                href="[[LINK_TO_ACCOUNT_NOTIFICATIONS]]"
+                                href="$accountNotificationLink"
                                 >Click
                                 <span style="color: #005a8e"><u>here</u></span>
                                 to manage your notifications setting.</a
diff --git a/controller-server/src/main/resources/mail/notification-message.vm b/controller-server/src/main/resources/mail/notification-message.vm
new file mode 100644
index 00000000000..29673d38420
--- /dev/null
+++ b/controller-server/src/main/resources/mail/notification-message.vm
@@ -0,0 +1,6 @@
+<p>
+    $esc.html($notificationHeader):
+</p>
+#foreach( $i in $notificationItems )
+<p>$i</p>
+#end
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java
index 1251963f01c..f64ed3740d2 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java
@@ -75,7 +75,7 @@ public class NotifierTest {
         var mail = mailer.inbox(email.getEmailAddress()).get(0);
 
         assertEquals("[WARNING] Test package Vespa Notification for tenant1.default.default", mail.subject());
-        assertEquals(new String(NotifierTest.class.getResourceAsStream("/mail/notification.txt").readAllBytes()), mail.htmlMessage().get());
+        assertEquals(new String(NotifierTest.class.getResourceAsStream("/mail/notification.html").readAllBytes()), mail.htmlMessage().get());
     }
 
     @Test
diff --git a/controller-server/src/test/resources/mail/notification.txt b/controller-server/src/test/resources/mail/notification.html
index 35db37fbc12..c8d0037426b 100644
--- a/controller-server/src/test/resources/mail/notification.txt
+++ b/controller-server/src/test/resources/mail/notification.html
@@ -383,11 +383,12 @@
                     style="vertical-align: top"
                     width="100%"
                   >
-                    <tbody>
-                      <tr>
-                        <td
-                          align="left"
-                          style="
+
+<tbody>
+<tr>
+    <td
+            align="left"
+            style="
                             font-size: 0px;
                             padding: 0px 25px 0px 25px;
                             padding-top: 0px;
@@ -396,9 +397,9 @@
                             padding-left: 50px;
                             word-break: break-word;
                           "
-                        >
-                          <div
-                            style="
+    >
+        <div
+                style="
                               font-family: Open Sans, Helvetica, Arial,
                                 sans-serif;
                               font-size: 13px;
@@ -406,23 +407,23 @@
                               text-align: left;
                               color: #797e82;
                             "
-                          >
-                            <h1
-                              style="
+        >
+            <h1
+                    style="
                                 text-align: center;
                                 color: #000000;
                                 line-height: 32px;
                               "
-                            >
-                              Vespa Cloud Notifications
-                            </h1>
-                          </div>
-                        </td>
-                      </tr>
-                      <tr>
-                        <td
-                          align="left"
-                          style="
+            >
+                Vespa Cloud Notifications
+            </h1>
+        </div>
+    </td>
+</tr>
+<tr>
+    <td
+            align="left"
+            style="
                             font-size: 0px;
                             padding: 0px 25px 0px 25px;
                             padding-top: 0px;
@@ -431,9 +432,9 @@
                             padding-left: 50px;
                             word-break: break-word;
                           "
-                        >
-                          <div
-                            style="
+    >
+        <div
+                style="
                               font-family: Open Sans, Helvetica, Arial,
                                 sans-serif;
                               font-size: 13px;
@@ -441,51 +442,54 @@
                               text-align: left;
                               color: #797e82;
                             "
-                          >
-                            <p>
-                              There are problems with tests for default.default:
-                            </p>
-                            <p>Test package has production tests, but no production tests are declared in deployment.xml</p><p>See <a href="https://docs.vespa.ai/en/testing.html">https://docs.vespa.ai/en/testing.html</a> for details on how to write system tests for Vespa</p>
-                          </div>
-                        </td>
-                      </tr>
-                      <tr>
-                        <td
-                          align="center"
-                          vertical-align="middle"
-                          style="
+        >
+
+<p>
+    There are problems with tests for default.default:
+</p>
+<p>Test package has production tests, but no production tests are declared in deployment.xml</p>
+<p>See <a href="https://docs.vespa.ai/en/testing.html">https://docs.vespa.ai/en/testing.html</a> for details on how to write system tests for Vespa</p>
+
+        </div>
+    </td>
+</tr>
+<tr>
+    <td
+            align="center"
+            vertical-align="middle"
+            style="
                             font-size: 0px;
                             padding: 10px 25px;
                             padding-top: 20px;
                             padding-bottom: 20px;
                             word-break: break-word;
                           "
-                        >
-                          <table
-                            border="0"
-                            cellpadding="0"
-                            cellspacing="0"
-                            role="presentation"
-                            style="border-collapse: separate; line-height: 100%"
-                          >
-                            <tbody>
-                              <tr>
-                                <td
-                                  align="center"
-                                  bgcolor="#005A8E"
-                                  role="presentation"
-                                  style="
+    >
+        <table
+                border="0"
+                cellpadding="0"
+                cellspacing="0"
+                role="presentation"
+                style="border-collapse: separate; line-height: 100%"
+        >
+            <tbody>
+            <tr>
+                <td
+                        align="center"
+                        bgcolor="#005A8E"
+                        role="presentation"
+                        style="
                                     border: none;
                                     border-radius: 100px;
                                     cursor: auto;
                                     mso-padding-alt: 15px 25px 15px 25px;
                                     background: #005a8e;
                                   "
-                                  valign="middle"
-                                >
-                                  <a
-                                    href="https://dashboard.tld/tenant/tenant1/application/default/prod/instance"
-                                    style="
+                        valign="middle"
+                >
+                    <a
+                            href="https://dashboard.tld/tenant/tenant1/application/default/prod/instance"
+                            style="
                                       display: inline-block;
                                       background: #005a8e;
                                       color: #ffffff;
@@ -501,20 +505,20 @@
                                       mso-padding-alt: 0px;
                                       border-radius: 100px;
                                     "
-                                    target="_blank"
-                                    ><b style="font-weight: 700"
-                                      ><b style="font-weight: 700"
-                                        >Go to Console</b
-                                      ></b
-                                    ></a
-                                  >
-                                </td>
-                              </tr>
-                            </tbody>
-                          </table>
-                        </td>
-                      </tr>
-                    </tbody>
+                            target="_blank"
+                    ><b style="font-weight: 700"
+                    ><b style="font-weight: 700"
+                    >Go to Console</b
+                    ></b
+                    ></a
+                    >
+                </td>
+            </tr>
+            </tbody>
+        </table>
+    </td>
+</tr>
+</tbody>
                   </table>
                 </div>
                 <!--[if mso | IE]></td></tr></table><![endif]-->
diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml
index 574b8c2f716..0b2acfe29ee 100644
--- a/dependency-versions/pom.xml
+++ b/dependency-versions/pom.xml
@@ -78,8 +78,11 @@
         <bouncycastle.vespa.version>1.76</bouncycastle.vespa.version>
         <byte-buddy.vespa.version>1.14.9</byte-buddy.vespa.version>
         <checker-qual.vespa.version>3.38.0</checker-qual.vespa.version>
+        <commons-beanutils.vespa.version>1.9.4</commons-beanutils.vespa.version>
         <commons-codec.vespa.version>1.16.0</commons-codec.vespa.version>
+        <commons-collections.vespa.version>3.2.2</commons-collections.vespa.version>
         <commons-csv.vespa.version>1.10.0</commons-csv.vespa.version>
+        <commons-digester.vespa.version>3.2</commons-digester.vespa.version>
         <commons-exec.vespa.version>1.3</commons-exec.vespa.version>
         <commons-io.vespa.version>2.14.0</commons-io.vespa.version>
         <commons-lang3.vespa.version>3.13.0</commons-lang3.vespa.version>
@@ -126,6 +129,8 @@
         <spifly.vespa.version>1.3.6</spifly.vespa.version>
         <snappy.vespa.version>1.1.10.5</snappy.vespa.version>
         <surefire.vespa.version>3.1.2</surefire.vespa.version>
+        <velocity.vespa.version>2.3</velocity.vespa.version>
+        <velocity.tools.vespa.version>3.1</velocity.tools.vespa.version>
         <wiremock.vespa.version>3.2.0</wiremock.vespa.version>
         <xerces.vespa.version>2.12.2</xerces.vespa.version>
         <zero-allocation-hashing.vespa.version>0.16</zero-allocation-hashing.vespa.version>
diff --git a/parent/pom.xml b/parent/pom.xml
index aec0f5b88fe..aed5fe071f3 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -933,7 +933,12 @@
             <dependency>
                 <groupId>org.apache.velocity</groupId>
                 <artifactId>velocity-engine-core</artifactId>
-                <version>2.3</version>
+                <version>${velocity.vespa.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.velocity.tools</groupId>
+                <artifactId>velocity-tools-generic</artifactId>
+                <version>${velocity.tools.vespa.version}</version>
             </dependency>
             <dependency>
                 <groupId>org.apiguardian</groupId>
diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt
index 619b032d24f..7f6e29f6957 100644
--- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt
+++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt
@@ -20,6 +20,7 @@ com.fasterxml.jackson.core:jackson-databind:${jackson-databind.vespa.version}
 com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jackson2.vespa.version}
 com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${jackson2.vespa.version}
 com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jackson2.vespa.version}
+com.github.cliftonlabs:json-simple:${findbugs.vespa.version}
 com.github.luben:zstd-jni:${luben.zstd.vespa.version}
 com.github.spotbugs:spotbugs-annotations:3.1.9
 com.google.code.findbugs:jsr305:${findbugs.vespa.version}
@@ -43,8 +44,10 @@ com.yahoo.athenz:athenz-zms-core:${athenz.vespa.version}
 com.yahoo.athenz:athenz-zpe-java-client:${athenz.vespa.version}
 com.yahoo.athenz:athenz-zts-core:${athenz.vespa.version}
 com.yahoo.rdl:rdl-java:1.5.4
+commons-beanutils:commons-beanutils:${commons-beanutils.vespa.version}
 commons-cli:commons-cli:1.5.0
 commons-codec:commons-codec:${commons-codec.vespa.version}
+commons-collections:commons-collections:${commons-collections.vespa.version}
 commons-fileupload:commons-fileupload:1.5
 commons-io:commons-io:${commons-io.vespa.version}
 commons-logging:commons-logging:${commons-logging.vespa.version}
@@ -87,6 +90,7 @@ org.antlr:antlr4-runtime:${antlr4.vespa.version}
 org.apache.aries.spifly:org.apache.aries.spifly.dynamic.bundle:${spifly.vespa.version}
 org.apache.commons:commons-compress:${commons-compress.vespa.version}
 org.apache.commons:commons-csv:${commons-csv.vespa.version}
+org.apache.commons:commons-digester3:${commons-digester.vespa.version}
 org.apache.commons:commons-exec:${commons-exec.vespa.version}
 org.apache.commons:commons-lang3:${commons-lang3.vespa.version}
 org.apache.commons:commons-math3:${commons.math3.vespa.version}
@@ -119,7 +123,8 @@ org.apache.maven:maven-project:2.2.1
 org.apache.maven:maven-repository-metadata:${maven-core.vespa.version}
 org.apache.maven:maven-settings:${maven-core.vespa.version}
 org.apache.opennlp:opennlp-tools:${opennlp.vespa.version}
-org.apache.velocity:velocity-engine-core:2.3
+org.apache.velocity.tools:velocity-tools-generic:${velocity.tools.vespa.version}
+org.apache.velocity:velocity-engine-core:${velocity.vespa.version}
 org.apache.yetus:audience-annotations:0.12.0
 org.apache.zookeeper:zookeeper-jute:${zookeeper.client.vespa.version}
 org.apache.zookeeper:zookeeper-jute:3.8.1