Merge branch 'master' into bratseth/typecheck-all

60 files changed, 2174 insertions, 66 deletions

diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java
index 706f797cd2c..beff50b52c6 100644
--- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java
+++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java
@@ -19,6 +19,7 @@ import java.security.cert.Certificate;
 import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
@@ -37,14 +38,15 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements
     // TODO Make expiry and update frequency configurable parameters
     private static final Duration CERTIFICATE_EXPIRY_TIME = Duration.ofDays(30);
     private static final Duration CERTIFICATE_UPDATE_PERIOD = Duration.ofDays(7);
-    private static final String DUMMY_PASSWORD = "athenz";
+    private static final String CERTIFICATE_ALIAS = "athenz";
+    private static final String CERTIFICATE_PASSWORD = "athenz";
 
     private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
     private final AthenzCertificateClient certificateClient;
     private final KeyProvider keyProvider;
     private final AthenzProviderServiceConfig.Zones zoneConfig;
     private final AtomicBoolean alreadyConfigured = new AtomicBoolean();
-    private KeyStore initialKeyStore;
+    private volatile KeyStore currentKeyStore;
 
     @Inject
     public AthenzSslKeyStoreConfigurator(KeyProvider keyProvider,
@@ -54,7 +56,7 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements
         this.certificateClient = new AthenzCertificateClient(config, zoneConfig);
         this.keyProvider = keyProvider;
         this.zoneConfig = zoneConfig;
-        this.initialKeyStore = downloadCertificate(keyProvider, certificateClient, zoneConfig);
+        this.currentKeyStore = downloadCertificate(keyProvider, certificateClient, zoneConfig);
     }
 
     @Override
@@ -62,8 +64,7 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements
         if (alreadyConfigured.getAndSet(true)) { // For debugging purpose of SslKeyStoreConfigurator interface
             throw new IllegalStateException("Already configured. configure() can only be called once.");
         }
-        sslKeyStoreContext.updateKeyStore(initialKeyStore, DUMMY_PASSWORD);
-        initialKeyStore = null;
+        sslKeyStoreContext.updateKeyStore(currentKeyStore, CERTIFICATE_PASSWORD);
         scheduler.scheduleAtFixedRate(new AthenzCertificateUpdater(sslKeyStoreContext),
                                       CERTIFICATE_UPDATE_PERIOD.toMinutes()/*initial delay*/,
                                       CERTIFICATE_UPDATE_PERIOD.toMinutes(),
@@ -80,6 +81,12 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements
         }
     }
 
+    Instant getKeyStoreExpiry() throws KeyStoreException {
+        X509Certificate certificate = (X509Certificate) currentKeyStore.getCertificate(CERTIFICATE_ALIAS);
+        return certificate.getNotAfter().toInstant();
+    }
+
+
     private static KeyStore downloadCertificate(KeyProvider keyProvider,
                                                 AthenzCertificateClient certificateClient,
                                                 AthenzProviderServiceConfig.Zones zoneConfig) {
@@ -90,7 +97,8 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements
 
             KeyStore keyStore = KeyStore.getInstance("JKS");
             keyStore.load(null);
-            keyStore.setKeyEntry("athenz", privateKey, DUMMY_PASSWORD.toCharArray(), new Certificate[]{certificate});
+            keyStore.setKeyEntry(
+                    CERTIFICATE_ALIAS, privateKey, CERTIFICATE_PASSWORD.toCharArray(), new Certificate[]{certificate});
             return keyStore;
         } catch (IOException | NoSuchAlgorithmException | CertificateException | KeyStoreException e) {
             throw new RuntimeException(e);
@@ -118,8 +126,8 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements
         public void run() {
             try {
                 log.log(LogLevel.INFO, "Updating Athenz certificate from ZTS");
-                KeyStore keyStore = downloadCertificate(keyProvider, certificateClient, zoneConfig);
-                sslKeyStoreContext.updateKeyStore(keyStore, DUMMY_PASSWORD);
+                currentKeyStore = downloadCertificate(keyProvider, certificateClient, zoneConfig);
+                sslKeyStoreContext.updateKeyStore(currentKeyStore, CERTIFICATE_PASSWORD);
                 log.log(LogLevel.INFO, "Athenz certificate reload successfully completed");
             } catch (Throwable e) {
                 log.log(LogLevel.ERROR, "Failed to update certificate from ZTS: " + e.getMessage(), e);
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslTrustStoreConfigurator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslTrustStoreConfigurator.java
index 8c8b5de2a30..7e24109a197 100644
--- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslTrustStoreConfigurator.java
+++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslTrustStoreConfigurator.java
@@ -25,6 +25,7 @@ import java.io.IOException;
 import java.math.BigInteger;
 import java.security.KeyPair;
 import java.security.KeyStore;
+import java.security.KeyStoreException;
 import java.security.Provider;
 import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
@@ -39,6 +40,7 @@ import java.util.logging.Logger;
 public class AthenzSslTrustStoreConfigurator implements SslTrustStoreConfigurator {
 
     private static final Logger log = Logger.getLogger(AthenzSslTrustStoreConfigurator.class.getName());
+    private static final String CERTIFICATE_ALIAS = "cfgselfsigned";
 
     private static final Provider provider = new BouncyCastleProvider();
     private final KeyStore trustStore;
@@ -56,6 +58,11 @@ public class AthenzSslTrustStoreConfigurator implements SslTrustStoreConfigurato
         log.log(LogLevel.INFO, "Configured JDisc trust store with self-signed certificate");
     }
 
+    Instant getTrustStoreExpiry() throws KeyStoreException {
+        X509Certificate certificate = (X509Certificate) trustStore.getCertificate(CERTIFICATE_ALIAS);
+        return certificate.getNotAfter().toInstant();
+    }
+
     private static KeyStore createTrustStore(KeyProvider keyProvider,
                                              ConfigserverConfig configserverConfig,
                                              AthenzProviderServiceConfig athenzProviderServiceConfig) {
@@ -67,7 +74,7 @@ public class AthenzSslTrustStoreConfigurator implements SslTrustStoreConfigurato
             try (FileInputStream in = new FileInputStream(athenzProviderServiceConfig.athenzCaTrustStore())) {
                 trustStore.load(in, "changeit".toCharArray());
             }
-            trustStore.setCertificateEntry("cfgselfsigned", selfSignedCertificate);
+            trustStore.setCertificateEntry(CERTIFICATE_ALIAS, selfSignedCertificate);
             return trustStore;
         } catch (Exception e) {
             throw new RuntimeException(e);
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CertificateExpiryMetricUpdater.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CertificateExpiryMetricUpdater.java
new file mode 100644
index 00000000000..cf734facf34
--- /dev/null
+++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CertificateExpiryMetricUpdater.java
@@ -0,0 +1,75 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.athenz.instanceproviderservice;
+
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.jdisc.Metric;
+
+import com.google.inject.Inject;
+
+import java.security.KeyStoreException;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * @author freva
+ */
+public class CertificateExpiryMetricUpdater extends AbstractComponent {
+
+    private static final Duration METRIC_REFRESH_PERIOD = Duration.ofMinutes(5);
+    private static final String NODE_CA_CERT_METRIC_NAME = "node-ca-cert.expiry.seconds";
+    private static final String ATHENZ_CONFIGSERVER_CERT_METRIC_NAME = "athenz-configserver-cert.expiry.seconds";
+
+    private final Logger logger = Logger.getLogger(CertificateExpiryMetricUpdater.class.getName());
+    private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
+    private final Metric metric;
+    private final AthenzSslKeyStoreConfigurator keyStoreConfigurator;
+    private final AthenzSslTrustStoreConfigurator trustStoreConfigurator;
+
+    @Inject
+    public CertificateExpiryMetricUpdater(Metric metric,
+                                          AthenzSslKeyStoreConfigurator keyStoreConfigurator,
+                                          AthenzSslTrustStoreConfigurator trustStoreConfigurator) {
+        this.metric = metric;
+        this.keyStoreConfigurator = keyStoreConfigurator;
+        this.trustStoreConfigurator = trustStoreConfigurator;
+
+
+        scheduler.scheduleAtFixedRate(this::updateMetrics,
+                30/*initial delay*/,
+                METRIC_REFRESH_PERIOD.getSeconds(),
+                TimeUnit.SECONDS);
+    }
+
+    @Override
+    public void deconstruct() {
+        try {
+            scheduler.shutdownNow();
+            scheduler.awaitTermination(30, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+            throw new RuntimeException("Failed to shutdown certificate expiry metrics updater on time", e);
+        }
+    }
+
+    private void updateMetrics() {
+        Instant now = Instant.now();
+
+        try {
+            Duration keyStoreExpiry = Duration.between(now, keyStoreConfigurator.getKeyStoreExpiry());
+            metric.set(ATHENZ_CONFIGSERVER_CERT_METRIC_NAME, keyStoreExpiry.getSeconds(), null);
+        } catch (KeyStoreException e) {
+            logger.log(Level.WARNING, "Failed to update key store expiry metric", e);
+        }
+
+        try {
+            Duration trustStoreExpiry = Duration.between(now, trustStoreConfigurator.getTrustStoreExpiry());
+            metric.set(NODE_CA_CERT_METRIC_NAME, trustStoreExpiry.getSeconds(), null);
+        } catch (KeyStoreException e) {
+            logger.log(Level.WARNING, "Failed to update trust store expiry metric", e);
+        }
+    }
+}
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java
index 60524fbca8d..a8e1256e032 100644
--- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java
+++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java
@@ -111,8 +111,8 @@ public class FilesApplicationFile extends ApplicationFile {
             file.getParentFile().mkdirs();
         }
         try {
-            String data = com.yahoo.io.IOUtils.readAll(input);
             String status = file.exists() ? ApplicationFile.ContentStatusChanged : ApplicationFile.ContentStatusNew;
+            String data = com.yahoo.io.IOUtils.readAll(input);
             IOUtils.writeFile(file, data, false);
             writeMetaFile(data, status);
         } catch (IOException e) {
@@ -122,6 +122,21 @@ public class FilesApplicationFile extends ApplicationFile {
     }
 
     @Override
+    public ApplicationFile appendFile(String value) {
+        if (file.getParentFile() != null) {
+            file.getParentFile().mkdirs();
+        }
+        try {
+            String status = file.exists() ? ApplicationFile.ContentStatusChanged : ApplicationFile.ContentStatusNew;
+            IOUtils.writeFile(file, value, true);
+            writeMetaFile(value, status);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        return this;
+    }
+
+    @Override
     public List<ApplicationFile> listFiles(final PathFilter filter) {
         List<ApplicationFile> files = new ArrayList<>();
         if (!file.isDirectory()) {
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java
index 0384a5c7a1c..33b7807aac5 100644
--- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java
+++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java
@@ -75,6 +75,13 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> {
     public abstract ApplicationFile writeFile(Reader input);
 
     /**
+     * Appends the given string to this text file.
+     *
+     * @return this
+     */
+    public abstract ApplicationFile appendFile(String value);
+
+    /**
      * List the files under this directory. If this is file, an empty list is returned.
      * Only immediate files/subdirectories are returned.
      *
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
index 77dc03ac4d1..d1a29271014 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
@@ -749,7 +749,9 @@ public class RankProfile implements Serializable, Cloneable {
     public TypeContext typeContext(QueryProfileRegistry queryProfiles) {
         MapTypeContext context = new MapTypeContext();
 
-        // Add constants
+        // Add small constants
+        getConstants().forEach((k, v) -> context.setType(FeatureNames.asConstantFeature(k), v.type()));
+        // Add large constants
         getSearch().getRankingConstants().forEach((k, v) -> context.setType(FeatureNames.asConstantFeature(k), v.getTensorType()));
 
         // Add attributes
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java
index 495ca7dd14a..2b997aa25f2 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java
@@ -2,6 +2,7 @@
 package com.yahoo.searchdefinition.expressiontransforms;
 
 import com.google.common.base.Joiner;
+import com.yahoo.collections.Pair;
 import com.yahoo.config.application.api.ApplicationFile;
 import com.yahoo.config.application.api.ApplicationPackage;
 import com.yahoo.config.model.application.provider.FilesApplicationPackage;
@@ -11,6 +12,9 @@ import com.yahoo.search.query.profile.QueryProfileRegistry;
 import com.yahoo.searchdefinition.RankProfile;
 import com.yahoo.searchdefinition.RankingConstant;
 import com.yahoo.searchlib.rankingexpression.RankingExpression;
+import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue;
+import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
+import com.yahoo.searchlib.rankingexpression.evaluation.Value;
 import com.yahoo.searchlib.rankingexpression.integration.tensorflow.TensorFlowImporter;
 import com.yahoo.searchlib.rankingexpression.integration.tensorflow.TensorFlowModel;
 import com.yahoo.searchlib.rankingexpression.integration.tensorflow.TensorFlowModel.Signature;
@@ -25,11 +29,13 @@ import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.TensorType;
 import com.yahoo.tensor.serialization.TypedBinaryFormat;
 
+import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
 import java.io.StringReader;
 import java.io.UncheckedIOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -92,16 +98,21 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil
         verifyRequiredMacros(expression, model.requiredMacros(), profile, queryProfiles);
         store.writeConverted(expression);
 
-        model.constants().forEach((k, v) -> transformConstant(store, profile, k, v));
+        model.smallConstants().forEach((k, v) -> transformSmallConstant(store, profile, k, v));
+        model.largeConstants().forEach((k, v) -> transformLargeConstant(store, profile, k, v));
 
         return expression.getRoot();
     }
 
     private ExpressionNode transformFromStoredModel(ModelStore store, RankProfile profile) {
-        for (RankingConstant constant : store.readRankingConstants()) {
+        for (Pair<String, Tensor> constant : store.readSmallConstants())
+            profile.addConstant(constant.getFirst(), asValue(constant.getSecond()));
+
+        for (RankingConstant constant : store.readLargeConstants()) {
             if ( ! profile.getSearch().getRankingConstants().containsKey(constant.getName()))
                 profile.getSearch().addRankingConstant(constant);
         }
+
         return store.readConverted().getRoot();
     }
 
@@ -158,8 +169,13 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil
         }
     }
 
-    private void transformConstant(ModelStore store, RankProfile profile, String constantName, Tensor constantValue) {
-        Path constantPath = store.writeConstant(constantName, constantValue);
+    private void transformSmallConstant(ModelStore store, RankProfile profile, String constantName, Tensor constantValue) {
+        store.writeSmallConstant(constantName, constantValue);
+        profile.addConstant(constantName, asValue(constantValue));
+    }
+
+    private void transformLargeConstant(ModelStore store, RankProfile profile, String constantName, Tensor constantValue) {
+        Path constantPath = store.writeLargeConstant(constantName, constantValue);
 
         if ( ! profile.getSearch().getRankingConstants().containsKey(constantName)) {
             log.info("Adding constant '" + constantName + "' of type " + constantValue.type());
@@ -218,6 +234,13 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil
         }
     }
 
+    private Value asValue(Tensor tensor) {
+        if (tensor.type().rank() == 0)
+            return new DoubleValue(tensor.asDouble()); // the backend gets offended by dimensionless tensors
+        else
+            return new TensorValue(tensor);
+    }
+
     /**
      * Provides read/write access to the correct directories of the application package given by the feature arguments
      */
@@ -272,13 +295,13 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil
         }
 
         /**
-         * Reads the information about all the constants stored in the application package
+         * Reads the information about all the large (aka ranking) constants stored in the application package
          * (the constant value itself is replicated with file distribution).
          */
-        public List<RankingConstant> readRankingConstants() {
+        public List<RankingConstant> readLargeConstants() {
             try {
                 List<RankingConstant> constants = new ArrayList<>();
-                for (ApplicationFile constantFile : application.getFile(arguments.rankingConstantsPath()).listFiles()) {
+                for (ApplicationFile constantFile : application.getFile(arguments.largeConstantsPath()).listFiles()) {
                     String[] parts = IOUtils.readAll(constantFile.createReader()).split(":");
                     constants.add(new RankingConstant(parts[0], TensorType.fromSpec(parts[1]), parts[2]));
                 }
@@ -295,25 +318,63 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil
          *
          * @return the path to the stored constant, relative to the application package root
          */
-        public Path writeConstant(String name, Tensor constant) {
+        public Path writeLargeConstant(String name, Tensor constant) {
             Path constantsPath = ApplicationPackage.MODELS_GENERATED_DIR.append(arguments.modelPath).append("constants");
 
             // "tbf" ending for "typed binary format" - recognized by the nodes receiving the file:
             Path constantPath = constantsPath.append(name + ".tbf");
-            Path constantPathCorrected = constantPath;
-            if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(FilesApplicationPackage.preprocessed)
-                && ! constantPath.elements().contains(FilesApplicationPackage.preprocessed)) {
-                constantPathCorrected = Path.fromString(FilesApplicationPackage.preprocessed).append(constantPath);
-            }
 
             // Remember the constant in a file we replicate in ZooKeeper
-            application.getFile(arguments.rankingConstantsPath().append(name + ".constant"))
-                       .writeFile(new StringReader(name + ":" + constant.type() + ":" + constantPathCorrected));
+            application.getFile(arguments.largeConstantsPath().append(name + ".constant"))
+                       .writeFile(new StringReader(name + ":" + constant.type() + ":" + correct(constantPath)));
 
             // Write content explicitly as a file on the file system as this is distributed using file distribution
             createIfNeeded(constantsPath);
             IOUtils.writeFile(application.getFileReference(constantPath), TypedBinaryFormat.encode(constant));
-            return constantPathCorrected;
+            return correct(constantPath);
+        }
+
+        private List<Pair<String, Tensor>> readSmallConstants() {
+            try {
+                ApplicationFile file = application.getFile(arguments.smallConstantsPath());
+                if (!file.exists()) return Collections.emptyList();
+
+                List<Pair<String, Tensor>> constants = new ArrayList<>();
+                BufferedReader reader = new BufferedReader(file.createReader());
+                String line;
+                while (null != (line = reader.readLine())) {
+                    String[] parts = line.split("\t");
+                    String name = parts[0];
+                    TensorType type = TensorType.fromSpec(parts[1]);
+                    Tensor tensor = Tensor.from(type, parts[2]);
+                    constants.add(new Pair<>(name, tensor));
+                }
+                return constants;
+            }
+            catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        }
+
+        /**
+         * Append this constant to the single file used for small constants distributed as config
+         */
+        public void writeSmallConstant(String name, Tensor constant) {
+            // Secret file format for remembering constants:
+            application.getFile(arguments.smallConstantsPath()).appendFile(name + "\t" +
+                                                                           constant.type().toString() + "\t" +
+                                                                           constant.toString() + "\n");
+        }
+
+        /** Workaround for being constructed with the .preprocessed dir as root while later being used outside it */
+        private Path correct(Path path) {
+            if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(FilesApplicationPackage.preprocessed)
+                && ! path.elements().contains(FilesApplicationPackage.preprocessed)) {
+                return Path.fromString(FilesApplicationPackage.preprocessed).append(path);
+            }
+            else {
+                return path;
+            }
         }
 
         private void createIfNeeded(Path path) {
@@ -351,7 +412,13 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil
         public Optional<String> signature() { return signature; }
         public Optional<String> output() { return output; }
 
-        public Path rankingConstantsPath() {
+        /** Path to the small constants file */
+        public Path smallConstantsPath() {
+            return ApplicationPackage.MODELS_GENERATED_DIR.append(modelPath).append("constants.txt");
+        }
+
+        /** Path to the large (ranking) constants directory */
+        public Path largeConstantsPath() {
             return ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR.append(modelPath).append("constants");
         }
 
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorTransformer.java
index 5255cdaeba1..0334012e8d9 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorTransformer.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorTransformer.java
@@ -183,7 +183,7 @@ public class TensorTransformer extends ExpressionTransformer<RankProfileTransfor
     }
 
     private void addIfConstant(ReferenceNode node, Context context, RankProfile profile) {
-        if (!node.getName().equals(ConstantTensorTransformer.CONSTANT)) {
+        if ( ! node.getName().equals(ConstantTensorTransformer.CONSTANT)) {
             return;
         }
         if (node.children().size() != 1) {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java
index 464772fc10d..58af8daf1b5 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java
@@ -399,6 +399,17 @@ public class RankingExpressionWithTensorFlowTestCase {
         }
 
         @Override
+        public ApplicationFile appendFile(String value) {
+            try {
+                IOUtils.writeFile(file, value, true);
+                return this;
+            }
+            catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        }
+
+        @Override
         public List<ApplicationFile> listFiles(PathFilter filter) {
             if ( ! isDirectory()) return Collections.emptyList();
             return Arrays.stream(file.listFiles()).filter(f -> filter.accept(Path.fromString(f.toString())))
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java
index 717fb88e5dc..affc2e03e2b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java
@@ -95,7 +95,6 @@ class ZKApplicationFile extends ApplicationFile {
 
     @Override
     public ApplicationFile writeFile(Reader input) {
-        // foo/bar/baz.txt
         String zkPath = getZKPath(path);
         try {
             String data = IOUtils.readAll(input);
@@ -112,6 +111,21 @@ class ZKApplicationFile extends ApplicationFile {
     }
 
     @Override
+    public ApplicationFile appendFile(String value) {
+        String zkPath = getZKPath(path);
+        String status = ContentStatusNew;
+        if (zkApp.exists(zkPath)) {
+            status = ContentStatusChanged;
+        }
+        String existingData = zkApp.getData(zkPath);
+        if (existingData == null)
+            existingData = "";
+        zkApp.putData(zkPath, existingData + value);
+        writeMetaFile(value, status);
+        return this;
+    }
+
+    @Override
     public List<ApplicationFile> listFiles(PathFilter filter) {
         String userPath = getZKPath(path);
         List<ApplicationFile> ret = new ArrayList<>();
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstancesReply.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstancesReply.java
index ff0c155460e..ebacafd75c4 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstancesReply.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstancesReply.java
@@ -15,4 +15,5 @@ public class InstancesReply {
     public Set<URI> globalRotations;
     public List<InstanceReference> instances;
     public String compileVersion;
+    public String rotationId;
 }
diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt
index 00ab5b347ea..8378af53098 100644
--- a/eval/CMakeLists.txt
+++ b/eval/CMakeLists.txt
@@ -29,6 +29,7 @@ vespa_define_module(
     src/tests/tensor/dense_tensor_builder
     src/tests/tensor/dense_tensor_function_optimizer
     src/tests/tensor/dense_xw_product_function
+    src/tests/tensor/vector_from_doubles_function
     src/tests/tensor/sparse_tensor_builder
     src/tests/tensor/tensor_address
     src/tests/tensor/tensor_conformance
diff --git a/eval/src/tests/tensor/vector_from_doubles_function/CMakeLists.txt b/eval/src/tests/tensor/vector_from_doubles_function/CMakeLists.txt
new file mode 100644
index 00000000000..5b2e47ec498
--- /dev/null
+++ b/eval/src/tests/tensor/vector_from_doubles_function/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(eval_vector_from_doubles_function_test_app TEST
+    SOURCES
+    vector_from_doubles_function_test.cpp
+    DEPENDS
+    vespaeval
+)
+vespa_add_test(NAME eval_vector_from_doubles_function_test_app COMMAND eval_vector_from_doubles_function_test_app)
diff --git a/eval/src/tests/tensor/vector_from_doubles_function/vector_from_doubles_function_test.cpp b/eval/src/tests/tensor/vector_from_doubles_function/vector_from_doubles_function_test.cpp
new file mode 100644
index 00000000000..0ba9871d672
--- /dev/null
+++ b/eval/src/tests/tensor/vector_from_doubles_function/vector_from_doubles_function_test.cpp
@@ -0,0 +1,164 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/log/log.h>
+LOG_SETUP("dense_dot_product_function_test");
+
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/eval/eval/tensor_function.h>
+#include <vespa/eval/eval/operation.h>
+#include <vespa/eval/eval/simple_tensor.h>
+#include <vespa/eval/eval/simple_tensor_engine.h>
+#include <vespa/eval/tensor/default_tensor_engine.h>
+#include <vespa/eval/tensor/dense/vector_from_doubles_function.h>
+#include <vespa/eval/tensor/dense/dense_tensor.h>
+#include <vespa/eval/tensor/dense/dense_tensor_builder.h>
+#include <vespa/eval/tensor/dense/dense_tensor_view.h>
+
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/stash.h>
+
+using namespace vespalib;
+using namespace vespalib::eval;
+using namespace vespalib::tensor;
+using namespace vespalib::eval::tensor_function;
+
+const TensorEngine &ref_engine = SimpleTensorEngine::ref();
+const TensorEngine &prod_engine = DefaultTensorEngine::ref();
+
+//-----------------------------------------------------------------------------
+// verify that optimize() works as expected
+
+template<typename OPT>
+bool treeContains(const TensorFunction &expr) {
+    using Child = TensorFunction::Child;
+    Child root(expr);
+    std::vector<Child::CREF> nodes({root});
+    for (size_t i = 0; i < nodes.size(); ++i) {
+        nodes[i].get().get().push_children(nodes);
+    }
+    for (const Child &child : nodes) {
+        if (as<OPT>(child.get())) {
+            return true;
+        }
+    }
+    return false;
+}
+
+const TensorFunction &optimize_fun(const Function &fun, const NodeTypes &node_types, Stash &stash) {
+    const TensorFunction &plain_fun = make_tensor_function(prod_engine, fun.root(), node_types, stash);
+    return prod_engine.optimize(plain_fun, stash);
+}
+
+std::vector<ValueType> extract_types(size_t n, const std::vector<TensorSpec> &input) {
+    std::vector<ValueType> vec;
+    for (const TensorSpec &spec : input) {
+        vec.push_back(ValueType::from_spec(spec.type()));
+    }
+    while (vec.size() < n) {
+        vec.push_back(ValueType::double_type());
+    }
+    return vec;
+}
+
+struct Context {
+    Stash stash;
+    Function function;
+    std::vector<TensorSpec> input;
+    std::vector<ValueType> input_types;
+    NodeTypes node_types;
+    const TensorFunction &optimized;
+
+    Context(const vespalib::string &expr, std::vector<TensorSpec> in)
+        : stash(),
+          function(Function::parse(expr)),
+          input(in),
+          input_types(extract_types(function.num_params(), input)),
+          node_types(function, input_types),
+          optimized(optimize_fun(function, node_types, stash))
+    {
+        EXPECT_EQUAL(actual(), expected());
+    }
+
+    ~Context() {}
+
+    struct Params : LazyParams {
+        std::vector<Value::UP> values;
+        Value &resolve(size_t idx, Stash &) const override {
+            return *values[idx];
+        }
+    };
+
+    Params gen_params(const TensorEngine &engine) {
+        Params p;
+        for (const TensorSpec &spec : input) {
+            p.values.emplace_back(engine.from_spec(spec));
+        }
+        while (p.values.size() < function.num_params()) {
+            double v = 1.0 + p.values.size();
+            p.values.emplace_back(std::make_unique<DoubleValue>(v));
+        }
+        return p;
+    }
+
+    TensorSpec actual() {
+        const LazyParams &params = gen_params(prod_engine);
+        InterpretedFunction prodIfun(prod_engine, optimized);
+        InterpretedFunction::Context prodIctx(prodIfun);
+        const Value &result = prodIfun.eval(prodIctx, params);
+        return prod_engine.to_spec(result);
+    }
+
+    TensorSpec expected() {
+        const LazyParams &params = gen_params(ref_engine);
+        InterpretedFunction refIfun(ref_engine, function, NodeTypes());
+        InterpretedFunction::Context refIctx(refIfun);
+        const Value &result = refIfun.eval(refIctx, params);
+        return ref_engine.to_spec(result);
+    }
+
+};
+
+//-----------------------------------------------------------------------------
+
+void verify_all_optimized(const vespalib::string &expr) {
+    Context context(expr, {});
+    EXPECT_TRUE(treeContains<VectorFromDoublesFunction>(context.optimized));
+    EXPECT_FALSE(treeContains<eval::tensor_function::Concat>(context.optimized));
+}
+
+TEST("require that multiple concats are optimized") {
+    TEST_DO(verify_all_optimized("concat(a,b,x)"));
+    TEST_DO(verify_all_optimized("concat(a,concat(b,concat(c,d,x),x),x)"));
+    TEST_DO(verify_all_optimized("concat(concat(concat(a,b,x),c,x),d,x)"));
+    TEST_DO(verify_all_optimized("concat(concat(a,b,x),concat(c,d,x),x)"));
+}
+
+//-----------------------------------------------------------------------------
+
+void verify_some_optimized(const vespalib::string &expr) {
+    Context context(expr, {});
+    EXPECT_TRUE(treeContains<VectorFromDoublesFunction>(context.optimized));
+    EXPECT_TRUE(treeContains<eval::tensor_function::Concat>(context.optimized));
+}
+
+TEST("require that concat along different dimension is not optimized") {
+    TEST_DO(verify_some_optimized("concat(concat(a,b,x),concat(c,d,x),y)"));
+}
+
+//-----------------------------------------------------------------------------
+
+TEST("require that concat of vector and double is not optimized") {
+    TensorSpec vecspec = TensorSpec("tensor(x[3])")
+                         .add({{"x", 0}}, 7.0)
+                         .add({{"x", 1}}, 11.0)
+                         .add({{"x", 2}}, 13.0);
+    TensorSpec dblspec = TensorSpec("double")
+                         .add({}, 19.0);
+    Context context("concat(a,b,x)", {vecspec, dblspec});
+    EXPECT_TRUE(treeContains<eval::tensor_function::Concat>(context.optimized));
+    EXPECT_FALSE(treeContains<VectorFromDoublesFunction>(context.optimized));
+}
+
+//-----------------------------------------------------------------------------
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp
index 0873d0341fa..5f8be58105a 100644
--- a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp
+++ b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp
@@ -9,6 +9,7 @@
 #include "dense/dense_tensor_builder.h"
 #include "dense/dense_dot_product_function.h"
 #include "dense/dense_xw_product_function.h"
+#include "dense/vector_from_doubles_function.h"
 #include <vespa/eval/eval/value.h>
 #include <vespa/eval/eval/tensor_spec.h>
 #include <vespa/eval/eval/simple_tensor_engine.h>
@@ -217,6 +218,7 @@ DefaultTensorEngine::optimize(const TensorFunction &expr, Stash &stash) const
     }
     while (!nodes.empty()) {
         const Child &child = nodes.back();
+        child.set(VectorFromDoublesFunction::optimize(child.get(), stash));
         child.set(DenseDotProductFunction::optimize(child.get(), stash));
         child.set(DenseXWProductFunction::optimize(child.get(), stash));
         nodes.pop_back();
diff --git a/eval/src/vespa/eval/tensor/dense/CMakeLists.txt b/eval/src/vespa/eval/tensor/dense/CMakeLists.txt
index 3bd81ff8df3..23cab0c5f79 100644
--- a/eval/src/vespa/eval/tensor/dense/CMakeLists.txt
+++ b/eval/src/vespa/eval/tensor/dense/CMakeLists.txt
@@ -10,5 +10,6 @@ vespa_add_library(eval_tensor_dense OBJECT
     dense_tensor_cells_iterator.cpp
     dense_tensor_view.cpp
     dense_tensor_reduce.cpp
+    vector_from_doubles_function.cpp
     mutable_dense_tensor_view.cpp
 )
diff --git a/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp b/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp
new file mode 100644
index 00000000000..445b08ab114
--- /dev/null
+++ b/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp
@@ -0,0 +1,110 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "vector_from_doubles_function.h"
+#include "dense_tensor.h"
+#include "dense_tensor_view.h"
+#include <vespa/eval/eval/operation.h>
+#include <vespa/eval/eval/value.h>
+#include <vespa/eval/tensor/tensor.h>
+
+namespace vespalib::tensor {
+
+using CellsRef = DenseTensorView::CellsRef;
+using eval::Value;
+using eval::ValueType;
+using eval::TensorFunction;
+using Child = eval::TensorFunction::Child;
+using eval::as;
+using namespace eval::tensor_function;
+using namespace eval::operation;
+
+namespace {
+
+void my_vector_from_doubles_op(eval::InterpretedFunction::State &state, uint64_t param) {
+    const auto *self = (const VectorFromDoublesFunction::Self *)(param);
+    ArrayRef<double> outputCells = state.stash.create_array<double>(self->resultSize);
+    for (size_t i = self->resultSize; i-- > 0; ) {
+        outputCells[i] = state.peek(0).as_double();
+        state.stack.pop_back();
+    }
+    const Value &result = state.stash.create<DenseTensorView>(self->resultType, outputCells);
+    state.stack.push_back(result);
+}
+
+size_t vector_size(const TensorFunction &child, const vespalib::string &dimension) {
+    if (child.result_type().is_double()) {
+        return 1;
+    }
+    if (auto vfd = as<VectorFromDoublesFunction>(child)) {
+        if (vfd->dimension() == dimension) {
+            return vfd->size();
+        }
+    }
+    return 0;
+}
+
+void flatten_into(const TensorFunction &child, std::vector<Child> &vec) {
+    if (child.result_type().is_double()) {
+        vec.push_back(child);
+    } else {
+        std::vector<Child::CREF> tmp;
+        child.push_children(tmp);
+        for (const Child &c : tmp) {
+            assert(c.get().result_type().is_double());
+            vec.push_back(c);
+        }
+    }
+}
+
+std::vector<Child> flatten(const TensorFunction &lhs, const TensorFunction &rhs) {
+    std::vector<Child> vec;
+    flatten_into(lhs, vec);
+    flatten_into(rhs, vec);
+    return vec;
+}
+
+} // namespace vespalib::tensor::<unnamed>
+
+
+VectorFromDoublesFunction::VectorFromDoublesFunction(std::vector<Child> children, const ValueType &res_type)
+    : TensorFunction(),
+      _self(res_type, children.size()),
+      _children(std::move(children))
+{
+}
+
+VectorFromDoublesFunction::~VectorFromDoublesFunction()
+{
+}
+
+void
+VectorFromDoublesFunction::push_children(std::vector<Child::CREF> &target) const
+{
+    for (const Child &c : _children) {
+        target.push_back(c);
+    }
+}
+
+eval::InterpretedFunction::Instruction
+VectorFromDoublesFunction::compile_self(Stash &) const
+{
+    return eval::InterpretedFunction::Instruction(my_vector_from_doubles_op, (uint64_t)&_self);
+}
+
+const TensorFunction &
+VectorFromDoublesFunction::optimize(const eval::TensorFunction &expr, Stash &stash)
+{
+    if (auto concat = as<Concat>(expr)) {
+        const vespalib::string &dimension = concat->dimension();
+        size_t a_size = vector_size(concat->lhs(), dimension);
+        size_t b_size = vector_size(concat->rhs(), dimension);
+        if ((a_size > 0) && (b_size > 0)) {
+            auto children = flatten(concat->lhs(), concat->rhs());
+            assert(children.size() == (a_size + b_size));
+            return stash.create<VectorFromDoublesFunction>(std::move(children), expr.result_type());
+        }
+    }
+    return expr;
+}
+
+} // namespace vespalib::tensor
diff --git a/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.h b/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.h
new file mode 100644
index 00000000000..417c60c2aca
--- /dev/null
+++ b/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.h
@@ -0,0 +1,37 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/eval/eval/tensor_function.h>
+
+namespace vespalib::tensor {
+
+/**
+ * Tensor function for a concat forming a vector from double values
+ */
+class VectorFromDoublesFunction : public eval::TensorFunction
+{
+public:
+    struct Self {
+        const eval::ValueType resultType;
+        size_t resultSize;
+        Self(const eval::ValueType &r, size_t n) : resultType(r), resultSize(n) {}
+    };
+private:
+    Self _self;
+    std::vector<Child> _children;
+    void add(const eval::TensorFunction &child);
+public:
+    VectorFromDoublesFunction(std::vector<Child> children, const eval::ValueType &res_type);
+    ~VectorFromDoublesFunction();
+    const eval::ValueType &result_type() const override { return _self.resultType; }
+    void push_children(std::vector<Child::CREF> &children) const override;
+    const vespalib::string &dimension() const {
+        return _self.resultType.dimensions()[0].name;
+    }
+    size_t size() const { return _self.resultSize; }
+    eval::InterpretedFunction::Instruction compile_self(Stash &stash) const override;
+    static const eval::TensorFunction &optimize(const eval::TensorFunction &expr, Stash &stash);
+};
+
+} // namespace vespalib::tensor
diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp
index 5530e468007..39a9fdefc39 100644
--- a/metrics/src/vespa/metrics/metricmanager.cpp
+++ b/metrics/src/vespa/metrics/metricmanager.cpp
@@ -150,11 +150,16 @@ MetricManager::removeMetricUpdateHook(UpdateHook& hook)
     LOG(warning, "Update hook not registered");
 }
 
+bool
+MetricManager::isInitialized() const {
+    return static_cast<bool>(_configHandle);
+}
+
 void
 MetricManager::init(const config::ConfigUri & uri, FastOS_ThreadPool& pool,
                     bool startThread)
 {
-    if (_configHandle.get()) {
+    if (isInitialized()) {
         throw vespalib::IllegalStateException(
                 "The metric manager have already been initialized. "
                 "It can only be initialized once.", VESPA_STRLOC);
@@ -164,12 +169,11 @@ MetricManager::init(const config::ConfigUri & uri, FastOS_ThreadPool& pool,
     _configHandle = _configSubscriber->subscribe<Config>(uri.getConfigId());
     _configSubscriber->nextConfig();
     configure(getMetricLock(), _configHandle->getConfig());
-    LOG(debug, "Starting worker thread, waiting for first "
-               "iteration to complete.");
+    LOG(debug, "Starting worker thread, waiting for first iteration to complete.");
     if (startThread) {
         Runnable::start(pool);
-            // Wait for first iteration to have completed, such that it is safe
-            // to access snapshots afterwards.
+        // Wait for first iteration to have completed, such that it is safe
+        // to access snapshots afterwards.
         vespalib::MonitorGuard sync(_waiter);
         while (_lastProcessedTime == 0) {
             sync.wait(1);
diff --git a/metrics/src/vespa/metrics/metricmanager.h b/metrics/src/vespa/metrics/metricmanager.h
index 7aeca264328..423eb41a787 100644
--- a/metrics/src/vespa/metrics/metricmanager.h
+++ b/metrics/src/vespa/metrics/metricmanager.h
@@ -273,6 +273,8 @@ public:
 
     MemoryConsumption::UP getMemoryConsumption(const MetricLockGuard & guard) const;
 
+    bool isInitialized() const;
+
 private:
     void takeSnapshots(const MetricLockGuard &, time_t timeToProcess);
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TestTaskContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java
index 1fe63e84605..6806e5096c5 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TestTaskContext.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java
@@ -1,9 +1,6 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-package com.yahoo.vespa.hosted.node.admin.task.util.file;
-
-import com.yahoo.vespa.hosted.node.admin.component.IdempotentTask;
-import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+package com.yahoo.vespa.hosted.node.admin.component;
 
 import java.util.ArrayList;
 import java.util.List;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java
index 3910398a040..611e2c32bcd 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java
@@ -24,4 +24,5 @@ public class FileAttributes {
     public String permissions() { return PosixFilePermissions.toString(attributes.permissions()); }
     public boolean isRegularFile() { return attributes.isRegularFile(); }
     public boolean isDirectory() { return attributes.isDirectory(); }
+    public long size() { return attributes.size(); }
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/InputStreamUtil.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/InputStreamUtil.java
new file mode 100644
index 00000000000..780102e9c9e
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/InputStreamUtil.java
@@ -0,0 +1,40 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.uncheck;
+
+/**
+ * @author hakonhall
+ */
+public class InputStreamUtil {
+    private final InputStream inputStream;
+
+    public InputStreamUtil(InputStream inputStream) {
+        this.inputStream = inputStream;
+    }
+
+    public InputStream getInputStream() {
+        return inputStream;
+    }
+
+    /**
+     * TODO: Replace usages with Java 9's InputStream::readAllBytes
+     */
+    byte[] readAllBytes() {
+        // According to https://stackoverflow.com/questions/309424/read-convert-an-inputstream-to-a-string
+        // all other implementations are much inferior to this in performance.
+
+        ByteArrayOutputStream result = new ByteArrayOutputStream();
+        byte[] buffer = new byte[1024];
+        int length;
+        while ((length = uncheck(() -> inputStream.read(buffer))) != -1) {
+            result.write(buffer, 0, length);
+        }
+
+        return result.toByteArray();
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2.java
new file mode 100644
index 00000000000..172203a281a
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2.java
@@ -0,0 +1,16 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+/**
+ * @author hakonhall
+ */
+interface ChildProcess2 extends AutoCloseable {
+    void waitForTermination();
+    int exitCode();
+    String getOutput();
+
+    /** Close/cleanup any resources held. Must not throw an exception. */
+    @Override
+    void close();
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java
new file mode 100644
index 00000000000..67020270a99
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java
@@ -0,0 +1,138 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.jdisc.Timer;
+import com.yahoo.log.LogLevel;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.uncheck;
+
+/**
+ * @author hakonhall
+ */
+public class ChildProcess2Impl implements ChildProcess2 {
+    private static final Logger logger = Logger.getLogger(ChildProcess2Impl.class.getName());
+
+    private final CommandLine commandLine;
+    private final ProcessApi2 process;
+    private final Path outputPath;
+    private final Timer timer;
+
+    public ChildProcess2Impl(CommandLine commandLine,
+                             ProcessApi2 process,
+                             Path outputPath,
+                             Timer timer) {
+        this.commandLine = commandLine;
+        this.process = process;
+        this.outputPath = outputPath;
+        this.timer = timer;
+    }
+
+    @Override
+    public void waitForTermination() {
+        Duration timeoutDuration = commandLine.getTimeout();
+        Instant timeout = timer.currentTime().plus(timeoutDuration);
+        long maxOutputBytes = commandLine.getMaxOutputBytes();
+
+        // How frequently do we want to wake up and check the output file size?
+        final Duration pollInterval = Duration.ofSeconds(10);
+
+        boolean hasTerminated = false;
+        while (!hasTerminated) {
+            Instant now = timer.currentTime();
+            long sleepPeriodMillis = pollInterval.toMillis();
+            if (now.plusMillis(sleepPeriodMillis).isAfter(timeout)) {
+                sleepPeriodMillis = Duration.between(now, timeout).toMillis();
+
+                if (sleepPeriodMillis <= 0) {
+                    gracefullyKill();
+                    throw new TimeoutChildProcessException(
+                            timeoutDuration, commandLine.toString(), getOutput());
+                }
+            }
+
+            try {
+                hasTerminated = process.waitFor(sleepPeriodMillis, TimeUnit.MILLISECONDS);
+            } catch (InterruptedException e) {
+                // Ignore, just loop around.
+                continue;
+            }
+
+            // Always check output file size to ensure we don't load too much into memory.
+            long sizeInBytes = uncheck(() -> Files.size(outputPath));
+            if (sizeInBytes > maxOutputBytes) {
+                gracefullyKill();
+                throw new LargeOutputChildProcessException(
+                        sizeInBytes, commandLine.toString(), getOutput());
+            }
+        }
+    }
+
+    @Override
+    public int exitCode() {
+        return process.exitValue();
+    }
+
+    @Override
+    public String getOutput() {
+        byte[] bytes = uncheck(() -> Files.readAllBytes(outputPath));
+        return new String(bytes, commandLine.getOutputEncoding());
+    }
+
+    @Override
+    public void close() {
+        try {
+            Files.delete(outputPath);
+        } catch (Throwable t) {
+            logger.log(LogLevel.WARNING, "Failed to delete " + outputPath, t);
+        }
+    }
+
+    Path getOutputPath() {
+        return outputPath;
+    }
+
+    private void gracefullyKill() {
+        process.destroy();
+
+        Duration maxWaitAfterSigTerm = commandLine.getSigTermGracePeriod();
+        Instant timeout = timer.currentTime().plus(maxWaitAfterSigTerm);
+        if (!waitForTermination(timeout)) {
+            process.destroyForcibly();
+
+            // If waiting for the process now takes a long time, it's probably a kernel issue
+            // or huge core is getting dumped.
+            Duration maxWaitAfterSigKill = commandLine.getSigKillGracePeriod();
+            if (!waitForTermination(timer.currentTime().plus(maxWaitAfterSigKill))) {
+                throw new UnkillableChildProcessException(
+                        maxWaitAfterSigTerm,
+                        maxWaitAfterSigKill,
+                        commandLine.toString(),
+                        getOutput());
+            }
+        }
+    }
+
+    /** @return true if process terminated, false on timeout. */
+    private boolean waitForTermination(Instant timeout) {
+        while (true) {
+            long waitDurationMillis = Duration.between(timer.currentTime(), timeout).toMillis();
+            if (waitDurationMillis <= 0) {
+                return false;
+            }
+
+            try {
+                return process.waitFor(waitDurationMillis, TimeUnit.MILLISECONDS);
+            } catch (InterruptedException e) {
+                // ignore
+            }
+        }
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java
new file mode 100644
index 00000000000..9f7aaab2060
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java
@@ -0,0 +1,79 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+/**
+ * Base class for child process related exceptions, with a util to build an error message
+ * that includes a large part of the output.
+ *
+ * @author hakonhall
+ */
+@SuppressWarnings("serial")
+public abstract class ChildProcessException extends RuntimeException {
+    private static final int MAX_OUTPUT_PREFIX = 200;
+    private static final int MAX_OUTPUT_SUFFIX = 200;
+    // Omitting a number of chars less than 10 or less than 10% would be ridiculous.
+    private static final int MAX_OUTPUT_SLACK = Math.max(10, (10 * (MAX_OUTPUT_PREFIX + MAX_OUTPUT_SUFFIX)) / 100);
+
+    /**
+     * An exception with a message of the following format:
+     *   Command 'COMMANDLINE' PROBLEM: stdout/stderr: 'OUTPUT'
+     *
+     * If the output of the terminated command is too large it will be sampled.
+     *
+     * @param problem            E.g. "terminated with exit code 1"
+     * @param commandLine        The command that failed in a concise (e.g. shell-like) format
+     * @param possiblyHugeOutput The output of the command
+     */
+    protected ChildProcessException(String problem, String commandLine, String possiblyHugeOutput) {
+        super(makeSnippet(problem, commandLine, possiblyHugeOutput));
+    }
+
+    protected ChildProcessException(RuntimeException cause,
+                                    String problem,
+                                    String commandLine,
+                                    String possiblyHugeOutput) {
+        super(makeSnippet(problem, commandLine, possiblyHugeOutput), cause);
+    }
+
+    private static String makeSnippet(String problem,
+                               String commandLine,
+                               String possiblyHugeOutput) {
+        return makeSnippet(
+                problem,
+                commandLine,
+                possiblyHugeOutput,
+                MAX_OUTPUT_PREFIX,
+                MAX_OUTPUT_SUFFIX,
+                MAX_OUTPUT_SLACK);
+    }
+
+    // Package-private instead of private for testing.
+    static String makeSnippet(String problem,
+                              String commandLine,
+                              String possiblyHugeOutput,
+                              int maxOutputPrefix,
+                              int maxOutputSuffix,
+                              int maxOutputSlack) {
+        StringBuilder stringBuilder = new StringBuilder()
+                .append("Command '")
+                .append(commandLine)
+                .append("' ")
+                .append(problem)
+                .append(": stdout/stderr: '");
+
+        if (possiblyHugeOutput.length() <= maxOutputPrefix + maxOutputSuffix + maxOutputSlack) {
+            stringBuilder.append(possiblyHugeOutput);
+        } else {
+            stringBuilder.append(possiblyHugeOutput.substring(0, maxOutputPrefix))
+                    .append("... [")
+                    .append(possiblyHugeOutput.length() - maxOutputPrefix - maxOutputSuffix)
+                    .append(" chars omitted] ...")
+                    .append(possiblyHugeOutput.substring(possiblyHugeOutput.length() - maxOutputSuffix));
+        }
+
+        stringBuilder.append("'");
+
+        return stringBuilder.toString();
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessFailureException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessFailureException.java
new file mode 100644
index 00000000000..5c6785a646c
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessFailureException.java
@@ -0,0 +1,15 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+/**
+ * The child process terminated with a non-zero exit code.
+ *
+ * @author hakonhall
+ */
+@SuppressWarnings("serial")
+public class ChildProcessFailureException extends ChildProcessException {
+    ChildProcessFailureException(int exitCode, String commandLine, String possiblyHugeOutput) {
+        super("terminated with exit code " + exitCode, commandLine, possiblyHugeOutput);
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java
new file mode 100644
index 00000000000..9a8ed19e0ac
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java
@@ -0,0 +1,271 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Predicate;
+import java.util.logging.Logger;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/**
+ * A CommandLine is used to specify and execute a shell-like program in a child process,
+ * and capture its output.
+ *
+ * @author hakonhall
+ */
+public class CommandLine {
+    private static Logger logger = Logger.getLogger(CommandLine.class.getName());
+    private static Pattern UNESCAPED_ARGUMENT_PATTERN = Pattern.compile("^[a-zA-Z0-9=@%/+:.,_-]+$");
+
+    /** The default timeout. See setTimeout() for details. */
+    public static final Duration DEFAULT_TIMEOUT = Duration.ofMinutes(10);
+
+    /** The default maximum number of output bytes. See setMaxOutputBytes() for details. */
+    public static final long DEFAULT_MAX_OUTPUT_BYTES = 1024 * 1024 * 1024; // 1 Gb
+
+    /**
+     * The default grace period after SIGTERM has been sent during a graceful kill.
+     * See setSigTermGracePeriod for details.
+     */
+    public static final Duration DEFAULT_SIGTERM_GRACE_PERIOD = Duration.ofMinutes(1);
+
+    /**
+     * The default grace period after SIGKILL has been sent during a graceful kill.
+     * See setSigKillGracePeriod for details.
+     */
+    public static final Duration DEFAULT_SIGKILL_GRACE_PERIOD = Duration.ofMinutes(30);
+
+    private final List<String> arguments = new ArrayList<>();
+    private final TaskContext taskContext;
+    private final ProcessFactory processFactory;
+
+    private boolean redirectStderrToStdoutInsteadOfDiscard = true;
+    private boolean executeSilentlyCalled = false;
+    private Charset outputEncoding = StandardCharsets.UTF_8;
+    private Duration timeout = DEFAULT_TIMEOUT;
+    private long maxOutputBytes = DEFAULT_MAX_OUTPUT_BYTES;
+    private Duration sigTermGracePeriod = DEFAULT_SIGTERM_GRACE_PERIOD;
+    private Duration sigKillGracePeriod = DEFAULT_SIGKILL_GRACE_PERIOD;
+    private Predicate<Integer> successfulExitCodePredicate = code -> code == 0;
+
+    public CommandLine(TaskContext taskContext, ProcessFactory processFactory) {
+        this.taskContext = taskContext;
+        this.processFactory = processFactory;
+    }
+
+    /** Add arguments to the command. The first argument in the first call to add() is the program. */
+    public CommandLine add(String... arguments) { return add(Arrays.asList(arguments)); }
+
+    /** Add arguments to the command. The first argument in the first call to add() is the program. */
+    public CommandLine add(Collection<String> arguments) {
+        this.arguments.addAll(arguments);
+        return this;
+    }
+
+    /** Add arguments by splitting arguments by space. */
+    public CommandLine addTokens(String arguments) {
+        return add(arguments.split(" "));
+    }
+
+    /**
+     * Execute a shell-like program in a child process:
+     *  - the program is recorded and logged as modifying the system, but see executeSilently().
+     *  - the program's stderr is redirected to stdout, but see discardStderr().
+     *  - the program's output is assumed to be UTF-8, but see setOutputEncoding().
+     *  - the program must terminate with exit code 0, but see ignoreExitCode().
+     *  - the output of the program will be accessible in the returned CommandResult.
+     *
+     * Footnote 1: As a safety measure the size of the output is capped, and the program is
+     * only allowed to execute up to a timeout. The defaults are set high so you typically do
+     * not have to worry about reaching these limits, but otherwise see setMaxOutputBytes()
+     * and setTimeout(), respectively.
+     *
+     * Footnote 2: If the child process is forced to be killed due to footnote 1, then
+     * setSigTermGracePeriod() and setSigKillGracePeriod() can be used to tweak how much time
+     * is given to the program to shut down. Again, the defaults should be reasonable.
+     */
+    public CommandResult execute() {
+        taskContext.recordSystemModification(logger, "Executing command: " + toString());
+        return doExecute();
+    }
+
+    /**
+     * Same as execute(), except it will not record the program as modifying the system.
+     *
+     * If the program is later found to have modified the system, or otherwise worthy of
+     * a record, call recordSilentExecutionAsSystemModification().
+     */
+    public CommandResult executeSilently() {
+        executeSilentlyCalled = true;
+        return doExecute();
+    }
+
+    /**
+     * Record an already executed executeSilently() as having modified the system.
+     * For instance with YUM it is not known until after a 'yum install' whether it
+     * modified the system.
+     */
+    public void recordSilentExecutionAsSystemModification() {
+        if (!executeSilentlyCalled) {
+            throw new IllegalStateException("executeSilently has not been called");
+        }
+        // Disallow multiple consecutive calls to this method without an intervening call
+        // to executeSilently().
+        executeSilentlyCalled = false;
+
+        taskContext.recordSystemModification(logger, "Executed command: " + toString());
+    }
+
+    /**
+     * The first argument of the command specifies the program and is either the program's
+     * filename (in case the environment variable PATH will be used to search for the program
+     * file) or a path with the last component being the program's filename.
+     *
+     * @return The filename of the program.
+     */
+    public String programName() {
+        if (arguments.isEmpty()) {
+            throw new IllegalStateException(
+                    "The program name cannot be determined yet as no arguments have been given");
+        }
+        String path = arguments.get(0);
+        int lastIndex = path.lastIndexOf('/');
+        if (lastIndex == -1) {
+            return path;
+        } else {
+            return path.substring(lastIndex + 1);
+        }
+    }
+
+    /** Returns a shell-like representation of the command. */
+    @Override
+    public String toString() {
+        String command = arguments.stream()
+                .map(CommandLine::maybeEscapeArgument)
+                .collect(Collectors.joining(" "));
+
+        // Note: Both of these cannot be confused with an argument since they would
+        // require escaping.
+        command += redirectStderrToStdoutInsteadOfDiscard ? " 2>&1" : " 2>/dev/null";
+
+        return command;
+    }
+
+
+    /**
+     * By default, stderr is redirected to stderr. This method will instead discard stderr.
+     */
+    public CommandLine discardStderr() {
+        this.redirectStderrToStdoutInsteadOfDiscard = false;
+        return this;
+    }
+
+    /**
+     * By default, a non-zero exit code will cause the command execution to fail. This method
+     * will instead ignore the exit code.
+     */
+    public CommandLine ignoreExitCode() {
+        this.successfulExitCodePredicate = code -> true;
+        return this;
+    }
+
+    /**
+     * By default, the output of the command is parsed as UTF-8. This method will set a
+     * different encoding.
+     */
+    public CommandLine setOutputEncoding(Charset outputEncoding) {
+        this.outputEncoding = outputEncoding;
+        return this;
+    }
+
+    /**
+     * By default, the command will be gracefully killed after DEFAULT_TIMEOUT. This method
+     * overrides that default.
+     */
+    public CommandLine setTimeout(Duration timeout) {
+        this.timeout = timeout;
+        return this;
+    }
+
+    /**
+     * By default, the command will be gracefully killed if it ever outputs more than
+     * DEFAULT_MAX_OUTPUT_BYTES. This method overrides that default.
+     */
+    public CommandLine setMaxOutputBytes(long maxOutputBytes) {
+        this.maxOutputBytes = maxOutputBytes;
+        return this;
+    }
+
+    /**
+     * By default, if the program needs to be gracefully killed it will wait up to
+     * DEFAULT_SIGTERM_GRACE_PERIOD for the program to exit after it has been killed with
+     * the SIGTERM signal.
+     */
+    public CommandLine setSigTermGracePeriod(Duration period) {
+        this.sigTermGracePeriod = period;
+        return this;
+    }
+
+    public CommandLine setSigKillGracePeriod(Duration period) {
+        this.sigTermGracePeriod = period;
+        return this;
+    }
+    // Accessor fields necessary for classes in this package. Could be public if necessary.
+    List<String> getArguments() { return Collections.unmodifiableList(arguments); }
+    boolean getRedirectStderrToStdoutInsteadOfDiscard() { return redirectStderrToStdoutInsteadOfDiscard; }
+    Predicate<Integer> getSuccessfulExitCodePredicate() { return successfulExitCodePredicate; }
+    Charset getOutputEncoding() { return outputEncoding; }
+    Duration getTimeout() { return timeout; }
+    long getMaxOutputBytes() { return maxOutputBytes; }
+    Duration getSigTermGracePeriod() { return sigTermGracePeriod; }
+    Duration getSigKillGracePeriod() { return sigKillGracePeriod; }
+
+    private CommandResult doExecute() {
+        try (ChildProcess2 child = processFactory.spawn(this)) {
+            child.waitForTermination();
+            int exitCode = child.exitCode();
+            if (!successfulExitCodePredicate.test(exitCode)) {
+                throw new ChildProcessFailureException(exitCode, toString(), child.getOutput());
+            }
+
+            String output = child.getOutput();
+            return new CommandResult(this, exitCode, output);
+        }
+    }
+
+    private static String maybeEscapeArgument(String argument) {
+        if (UNESCAPED_ARGUMENT_PATTERN.matcher(argument).matches()) {
+            return argument;
+        } else {
+            return escapeArgument(argument);
+        }
+    }
+
+    private static String escapeArgument(String argument) {
+        StringBuilder doubleQuoteEscaped = new StringBuilder(argument.length() + 10);
+
+        for (int i = 0; i < argument.length(); ++i) {
+            char c = argument.charAt(i);
+            switch (c) {
+                case '"':
+                case '\\':
+                    doubleQuoteEscaped.append("\\").append(c);
+                    break;
+                default:
+                    doubleQuoteEscaped.append(c);
+            }
+        }
+
+        return "\"" + doubleQuoteEscaped + "\"";
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandResult.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandResult.java
new file mode 100644
index 00000000000..e0e84242f78
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandResult.java
@@ -0,0 +1,92 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import java.util.List;
+import java.util.function.Function;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * A CommandResult is the result of the execution of a CommandLine.
+ *
+ * @author hakonhall
+ */
+public class CommandResult {
+    private static final Pattern NEWLINE = Pattern.compile("\\n");
+
+    private final CommandLine commandLine;
+    private final int exitCode;
+    private final String output;
+
+    CommandResult(CommandLine commandLine, int exitCode, String output) {
+        this.commandLine = commandLine;
+        this.exitCode = exitCode;
+        this.output = output;
+    }
+
+    public int getExitCode() {
+        return exitCode;
+    }
+
+    /** Returns the output with leading and trailing white-space removed. */
+    public String getOutput() { return output.trim(); }
+
+    public String getUntrimmedOutput() { return output; }
+
+    /** Returns the output lines of the command, omitting trailing empty lines. */
+    public List<String> getOutputLines() {
+        return getOutputLinesStream().collect(Collectors.toList());
+    }
+
+    /** Returns the output lines as a stream, omitting trailing empty lines. */
+    public Stream<String> getOutputLinesStream() {
+        if (output.isEmpty()) {
+            // For some reason an empty string => one-element list.
+            return Stream.empty();
+        }
+
+        // For some reason this removes trailing empty elements, but that's OK with us.
+        return NEWLINE.splitAsStream(output);
+    }
+
+    /**
+     * Map this CommandResult to an instance of type R.
+     *
+     * If a RuntimeException is thrown by the mapper, it is wrapped in an
+     * UnexpectedOutputException2 that includes a snippet of the output in the message.
+     *
+     * This method is intended to be used as part of the verification of the output.
+     */
+    public <R> R map(Function<CommandResult, R> mapper) {
+        try {
+            return mapper.apply(this);
+        } catch (RuntimeException e) {
+            throw new UnexpectedOutputException2(e, "Failed to map output", commandLine.toString(), output);
+        }
+    }
+
+    /**
+     * Map the output to an instance of type R according to mapper, wrapping any
+     * RuntimeException in UnexpectedOutputException2 w/output snippet. See map() for details.
+     */
+    public <R> R mapOutput(Function<String, R> mapper) { return map(result -> mapper.apply(result.getOutput())); }
+
+    /**
+     * Map each output line to an instance of type R according to mapper, wrapping any
+     * RuntimeException in UnexpectedOutputException2 w/output snippet. See map() for details.
+     */
+    public <R> List<R> mapEachLine(Function<String, R> mapper) {
+        return map(result -> result.getOutputLinesStream().map(mapper).collect(Collectors.toList()));
+    }
+
+    /**
+     * Convenience method for getting the CommandLine, whose execution resulted in
+     * this CommandResult instance.
+     *
+     * Warning: the CommandLine is mutable and may be changed by the caller of the execution
+     * through other references! This is just a convenience method for getting that instance.
+     */
+    public CommandLine getCommandLine() { return commandLine; }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/LargeOutputChildProcessException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/LargeOutputChildProcessException.java
new file mode 100644
index 00000000000..5c764757e84
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/LargeOutputChildProcessException.java
@@ -0,0 +1,15 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+/**
+ * Exception thrown if the output of the child process is larger than the maximum limit.
+ *
+ * @author hakonhall
+ */
+@SuppressWarnings("serial")
+public class LargeOutputChildProcessException extends ChildProcessException {
+    LargeOutputChildProcessException(long maxFileSize, String commandLine, String possiblyHugeOutput) {
+        super("output more than " + maxFileSize + " bytes", commandLine, possiblyHugeOutput);
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2.java
new file mode 100644
index 00000000000..124f319e932
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2.java
@@ -0,0 +1,17 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Process abstraction.
+ *
+ * @author hakonhall
+ */
+public interface ProcessApi2 {
+    boolean waitFor(long timeout, TimeUnit unit) throws InterruptedException;
+    int exitValue();
+    void destroy();
+    void destroyForcibly();
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2Impl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2Impl.java
new file mode 100644
index 00000000000..853558c38e6
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2Impl.java
@@ -0,0 +1,36 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author hakonhall
+ */
+public class ProcessApi2Impl implements ProcessApi2 {
+    private final Process process;
+
+    ProcessApi2Impl(Process process) {
+        this.process = process;
+    }
+
+    @Override
+    public boolean waitFor(long timeout, TimeUnit unit) throws InterruptedException {
+        return process.waitFor(timeout, unit);
+    }
+
+    @Override
+    public int exitValue() {
+        return process.exitValue();
+    }
+
+    @Override
+    public void destroy() {
+        process.destroy();
+    }
+
+    @Override
+    public void destroyForcibly() {
+        process.destroyForcibly();
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApiImpl.java
index e664a68aeff..3620ec9089e 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApiImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApiImpl.java
@@ -41,7 +41,6 @@ public class ProcessApiImpl implements ProcessApi {
 
     @Override
     public void close() {
-        // TODO: Should kill process if still alive?
         processOutputPath.toFile().delete();
     }
 
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactory.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactory.java
new file mode 100644
index 00000000000..3351563faf5
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactory.java
@@ -0,0 +1,10 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+/**
+ * @author hakonhall
+ */
+public interface ProcessFactory {
+    ChildProcess2 spawn(CommandLine commandLine);
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java
new file mode 100644
index 00000000000..1c7a60a13fc
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java
@@ -0,0 +1,89 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.jdisc.Timer;
+import com.yahoo.log.LogLevel;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.FileAttribute;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.List;
+import java.util.Set;
+import java.util.logging.Logger;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.uncheck;
+
+/**
+ * @author hakonhall
+ */
+public class ProcessFactoryImpl implements ProcessFactory {
+    private static final Logger logger = Logger.getLogger(ProcessFactoryImpl.class.getName());
+    private static final File DEV_NULL = new File("/dev/null");
+
+    private final ProcessStarter processStarter;
+    private final Timer timer;
+
+    ProcessFactoryImpl(ProcessStarter processStarter, Timer timer) {
+        this.processStarter = processStarter;
+        this.timer = timer;
+    }
+
+    @Override
+    public ChildProcess2Impl spawn(CommandLine commandLine) {
+        List<String> arguments = commandLine.getArguments();
+        if (arguments.isEmpty()) {
+            throw new IllegalArgumentException("No arguments specified - missing program to spawn");
+        }
+
+        ProcessBuilder processBuilder = new ProcessBuilder(arguments);
+
+        if (commandLine.getRedirectStderrToStdoutInsteadOfDiscard()) {
+            processBuilder.redirectErrorStream(true);
+        } else {
+            processBuilder.redirectError(ProcessBuilder.Redirect.to(DEV_NULL));
+        }
+
+        // The output is redirected to a temporary file because:
+        //  - We could read continuously from process.getInputStream, but that may block
+        //    indefinitely with a faulty program.
+        //  - If we don't read continuously from process.getInputStream, then because
+        //    the underlying channel may be a pipe, the child may be stopped because the pipe
+        //    is full.
+        //  - To honor the timeout, no API can be used that may end up blocking indefinitely.
+        //
+        // Therefore, we redirect the output to a file and use waitFor w/timeout. This also
+        // has the benefit of allowing for inspection of the file during execution, and
+        // allowing the inspection of the file if it e.g. gets too large to hold in-memory.
+
+        String temporaryFilePrefix =
+                ProcessFactoryImpl.class.getSimpleName() + "-" + commandLine.programName() + "-";
+
+        FileAttribute<Set<PosixFilePermission>> fileAttribute = PosixFilePermissions.asFileAttribute(
+                PosixFilePermissions.fromString("rw-------"));
+
+        Path temporaryFile = uncheck(() -> Files.createTempFile(
+                temporaryFilePrefix,
+                ".out",
+                fileAttribute));
+
+        try {
+            processBuilder.redirectOutput(temporaryFile.toFile());
+            ProcessApi2 process = processStarter.start(processBuilder);
+            return new ChildProcess2Impl(commandLine, process, temporaryFile, timer);
+        } catch (RuntimeException | Error throwable) {
+            try {
+                Files.delete(temporaryFile);
+            } catch (IOException ioException) {
+                logger.log(LogLevel.WARNING, "Failed to delete temporary file at " +
+                        temporaryFile, ioException);
+            }
+            throw throwable;
+        }
+
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarter.java
new file mode 100644
index 00000000000..0afd4c6ee37
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarter.java
@@ -0,0 +1,10 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+/**
+ * @author hakonhall
+ */
+public interface ProcessStarter {
+    ProcessApi2 start(ProcessBuilder processBuilder);
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java
new file mode 100644
index 00000000000..2694a2929c4
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java
@@ -0,0 +1,16 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.uncheck;
+
+/**
+ * @author hakonhall
+ */
+public class ProcessStarterImpl implements ProcessStarter {
+    @Override
+    public ProcessApi2 start(ProcessBuilder processBuilder) {
+        Process process = uncheck(() -> processBuilder.start());
+        return new ProcessApi2Impl(process);
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/Terminal.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/Terminal.java
new file mode 100644
index 00000000000..849099ab5ca
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/Terminal.java
@@ -0,0 +1,14 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+/**
+ * A Terminal is a light-weight terminal-like interface for executing shell-like programs.
+ *
+ * @author hakonhall
+ */
+public interface Terminal {
+    CommandLine newCommandLine(TaskContext taskContext);
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TerminalImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TerminalImpl.java
new file mode 100644
index 00000000000..8ec0d267f0d
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TerminalImpl.java
@@ -0,0 +1,26 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.jdisc.Timer;
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+/**
+ * @author hakonhall
+ */
+public class TerminalImpl implements Terminal {
+    private final ProcessFactory processFactory;
+
+    public TerminalImpl(Timer timer) {
+        this(new ProcessFactoryImpl(new ProcessStarterImpl(), timer));
+    }
+
+    /** For testing. */
+    public TerminalImpl(ProcessFactory processFactory) {
+        this.processFactory = processFactory;
+    }
+
+    @Override
+    public CommandLine newCommandLine(TaskContext taskContext) {
+        return new CommandLine(taskContext, processFactory);
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestChildProcess2.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestChildProcess2.java
new file mode 100644
index 00000000000..4e678522168
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestChildProcess2.java
@@ -0,0 +1,52 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import java.util.Optional;
+
+/**
+ * @author hakonhall
+ */
+public class TestChildProcess2 implements ChildProcess2 {
+    private final int exitCode;
+    private final String output;
+    private Optional<RuntimeException> exceptionToThrowInWaitForTermination = Optional.empty();
+    private boolean closeCalled = false;
+
+    public TestChildProcess2(int exitCode, String output) {
+        this.exitCode = exitCode;
+        this.output = output;
+    }
+
+    public void throwInWaitForTermination(RuntimeException e) {
+        this.exceptionToThrowInWaitForTermination = Optional.of(e);
+    }
+
+    @Override
+    public void waitForTermination() {
+        if (exceptionToThrowInWaitForTermination.isPresent()) {
+            throw exceptionToThrowInWaitForTermination.get();
+        }
+    }
+
+    @Override
+    public int exitCode() {
+        return exitCode;
+    }
+
+    @Override
+    public String getOutput() {
+        return output;
+    }
+
+    @Override
+    public void close() {
+        if (closeCalled) {
+            throw new IllegalStateException("close already called");
+        }
+        closeCalled = true;
+    }
+
+    public boolean closeCalled() {
+        return closeCalled;
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestProcessFactory.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestProcessFactory.java
new file mode 100644
index 00000000000..0586797d259
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestProcessFactory.java
@@ -0,0 +1,95 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ * @author hakonhall
+ */
+public class TestProcessFactory implements ProcessFactory {
+    private static class SpawnCall {
+        private final String commandDescription;
+        private final Function<CommandLine, ChildProcess2> callback;
+
+        private SpawnCall(String commandDescription,
+                          Function<CommandLine, ChildProcess2> callback) {
+            this.commandDescription = commandDescription;
+            this.callback = callback;
+        }
+    }
+    private final List<SpawnCall> expectedSpawnCalls = new ArrayList<>();
+    private final List<CommandLine> spawnCommandLines = new ArrayList<>();
+
+    /** Forward call to spawn() to callback. */
+    public TestProcessFactory interceptSpawn(String commandDescription,
+                                             Function<CommandLine, ChildProcess2> callback) {
+        expectedSpawnCalls.add(new SpawnCall(commandDescription, callback));
+        return this;
+    }
+
+    // Convenience method for the caller to avoid having to create a TestChildProcess2 instance.
+    public TestProcessFactory expectSpawn(String commandLineString, TestChildProcess2 toReturn) {
+        return interceptSpawn(
+                commandLineString,
+                commandLine -> defaultSpawn(commandLine, commandLineString, toReturn));
+    }
+
+    // Convenience method for the caller to avoid having to create a TestChildProcess2 instance.
+    public TestProcessFactory expectSpawn(String commandLine, int exitCode, String output) {
+        return expectSpawn(commandLine, new TestChildProcess2(exitCode, output));
+    }
+
+    /** Ignore the CommandLine passed to spawn(), just return successfully with the given output. */
+    public TestProcessFactory ignoreSpawn(String output) {
+        return interceptSpawn(
+                "[call index " + expectedSpawnCalls.size() + "]",
+                commandLine -> new TestChildProcess2(0, output));
+    }
+
+    public TestProcessFactory ignoreSpawn() {
+        return ignoreSpawn("");
+    }
+
+    public void verifyAllCommandsExecuted() {
+        if (spawnCommandLines.size() < expectedSpawnCalls.size()) {
+            int missingCommandIndex = spawnCommandLines.size();
+            throw new IllegalStateException("Command #" + missingCommandIndex +
+                    " never executed: " +
+                    expectedSpawnCalls.get(missingCommandIndex).commandDescription);
+        }
+    }
+
+    /**
+     * WARNING: CommandLine is mutable, and e.g. reusing a CommandLine for the next call
+     * would make the CommandLine in this list no longer reflect the original CommandLine.
+     */
+    public List<CommandLine> getMutableCommandLines() {
+        return spawnCommandLines;
+    }
+
+    @Override
+    public ChildProcess2 spawn(CommandLine commandLine) {
+        String commandLineString = commandLine.toString();
+        if (spawnCommandLines.size() + 1 > expectedSpawnCalls.size()) {
+            throw new IllegalStateException("Too many invocations: " + commandLineString);
+        }
+        spawnCommandLines.add(commandLine);
+
+        return expectedSpawnCalls.get(spawnCommandLines.size() - 1).callback.apply(commandLine);
+    }
+
+    private static ChildProcess2 defaultSpawn(CommandLine commandLine,
+                                              String expectedCommandLineString,
+                                              ChildProcess2 toReturn) {
+        String actualCommandLineString = commandLine.toString();
+        if (!Objects.equals(actualCommandLineString, expectedCommandLineString)) {
+            throw new IllegalArgumentException("Expected command line '" +
+                    expectedCommandLineString + "' but got '" + actualCommandLineString + "'");
+        }
+
+        return toReturn;
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestTerminal.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestTerminal.java
new file mode 100644
index 00000000000..57aeeb04532
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestTerminal.java
@@ -0,0 +1,67 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+import java.util.function.Function;
+
+/**
+ * @author hakonhall
+ */
+public class TestTerminal implements Terminal {
+    private final TerminalImpl realTerminal;
+    private final TestProcessFactory testProcessFactory = new TestProcessFactory();
+
+    public TestTerminal() {
+        this.realTerminal = new TerminalImpl(testProcessFactory);
+    }
+
+    /** Get the TestProcessFactory the terminal was started with. */
+    public TestProcessFactory getTestProcessFactory() { return testProcessFactory; }
+
+    /** Forward call to spawn() to callback. */
+    public TestTerminal interceptCommand(String commandDescription,
+                                         Function<CommandLine, ChildProcess2> callback) {
+        testProcessFactory.interceptSpawn(commandDescription, callback);
+        return this;
+    }
+
+    /** Wraps expectSpawn in TestProcessFactory, provided here as convenience. */
+    public TestTerminal expectCommand(String commandLine, TestChildProcess2 toReturn) {
+        testProcessFactory.expectSpawn(commandLine, toReturn);
+        return this;
+    }
+
+    /** Wraps expectSpawn in TestProcessFactory, provided here as convenience. */
+    public TestTerminal expectCommand(String commandLine, int exitCode, String output) {
+        testProcessFactory.expectSpawn(commandLine, new TestChildProcess2(exitCode, output));
+        return this;
+    }
+
+    /** Verifies command line matches commandLine, and returns successfully with output "". */
+    public TestTerminal expectCommand(String commandLine) {
+        expectCommand(commandLine, 0, "");
+        return this;
+    }
+
+    /** Wraps expectSpawn in TestProcessFactory, provided here as convenience. */
+    public TestTerminal ignoreCommand(String output) {
+        testProcessFactory.ignoreSpawn(output);
+        return this;
+    }
+
+    /** Wraps expectSpawn in TestProcessFactory, provided here as convenience. */
+    public TestTerminal ignoreCommand() {
+        testProcessFactory.ignoreSpawn();
+        return this;
+    }
+
+    public void verifyAllCommandsExecuted() {
+        testProcessFactory.verifyAllCommandsExecuted();
+    }
+
+    @Override
+    public CommandLine newCommandLine(TaskContext taskContext) {
+        return realTerminal.newCommandLine(taskContext);
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TimeoutChildProcessException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TimeoutChildProcessException.java
new file mode 100644
index 00000000000..df9e2dc3471
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TimeoutChildProcessException.java
@@ -0,0 +1,18 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import java.time.Duration;
+
+/**
+ * Exception thrown when a child process has taken too long to terminate, in case it has been
+ * forcibly killed.
+ *
+ * @author hakonhall
+ */
+@SuppressWarnings("serial")
+public class TimeoutChildProcessException extends ChildProcessException {
+    TimeoutChildProcessException(Duration timeout, String commandLine, String possiblyHugeOutput) {
+        super("timed out after " + timeout, commandLine, possiblyHugeOutput);
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnexpectedOutputException2.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnexpectedOutputException2.java
new file mode 100644
index 00000000000..82fae1aa70e
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnexpectedOutputException2.java
@@ -0,0 +1,26 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+/**
+ * @author hakonhall
+ */
+@SuppressWarnings("serial")
+public class UnexpectedOutputException2 extends ChildProcessException {
+    /**
+     * @param problem Problem description, e.g. "Output is not of the form ^NAME=VALUE$"
+     */
+    public UnexpectedOutputException2(String problem, String commandLine, String possiblyHugeOutput) {
+        super("output was not of the expected format: " + problem, commandLine, possiblyHugeOutput);
+    }
+
+    /**
+     * @param problem Problem description, e.g. "Output is not of the form ^NAME=VALUE$"
+     */
+    public UnexpectedOutputException2(RuntimeException cause,
+                                      String problem,
+                                      String commandLine,
+                                      String possiblyHugeOutput) {
+        super(cause, "output was not of the expected format: " + problem, commandLine, possiblyHugeOutput);
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnkillableChildProcessException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnkillableChildProcessException.java
new file mode 100644
index 00000000000..1da27dd853e
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnkillableChildProcessException.java
@@ -0,0 +1,21 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import java.time.Duration;
+
+/**
+ * @author hakonhall
+ */
+@SuppressWarnings("serial")
+public class UnkillableChildProcessException extends ChildProcessException {
+    public UnkillableChildProcessException(Duration waitForSigTerm,
+                                           Duration waitForSigKill,
+                                           String commandLine,
+                                           String possiblyHugeOutput) {
+        super("did not terminate even after SIGTERM, +" + waitForSigTerm +
+                ", SIGKILL, and +" + waitForSigKill,
+                commandLine,
+                possiblyHugeOutput);
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/time/TestTimer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/time/TestTimer.java
new file mode 100644
index 00000000000..beadeeed4a3
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/time/TestTimer.java
@@ -0,0 +1,29 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.time;
+
+import com.yahoo.jdisc.Timer;
+
+import java.time.Duration;
+
+/**
+ * @author hakonhall
+ */
+public class TestTimer implements Timer {
+    private Duration durationSinceEpoch = Duration.ZERO;
+
+    public void setMillis(long millisSinceEpoch) {
+        durationSinceEpoch = Duration.ofMillis(millisSinceEpoch);
+    }
+
+    public void advanceMillis(long millis) { advance(Duration.ofMillis(millis)); }
+    public void advanceSeconds(long seconds) { advance(Duration.ofSeconds(seconds)); }
+    public void advanceMinutes(long minutes) { advance(Duration.ofMinutes(minutes)); }
+    public void advance(Duration duration) {
+        durationSinceEpoch = durationSinceEpoch.plus(duration);
+    }
+
+    @Override
+    public long currentTimeMillis() {
+        return durationSinceEpoch.toMillis();
+    }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java
index 44868e17464..dbc2cc9a5d5 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java
@@ -2,6 +2,7 @@
 
 package com.yahoo.vespa.hosted.node.admin.task.util.file;
 
+import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext;
 import com.yahoo.vespa.test.file.TestFileSystem;
 import org.junit.Test;
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java
index 05662de3b95..a83f3bbe7d4 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java
@@ -2,6 +2,7 @@
 
 package com.yahoo.vespa.hosted.node.admin.task.util.file;
 
+import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext;
 import com.yahoo.vespa.test.file.TestFileSystem;
 import org.junit.Test;
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java
new file mode 100644
index 00000000000..1a88af8ad0f
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java
@@ -0,0 +1,147 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.jdisc.Timer;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.Instant;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author hakonhall
+ */
+public class ChildProcess2ImplTest {
+    private final FileSystem fileSystem = TestFileSystem.create();
+    private final Timer timer = mock(Timer.class);
+    private final CommandLine commandLine = mock(CommandLine.class);
+    private final ProcessApi2 processApi = mock(ProcessApi2.class);
+    private Path temporaryFile;
+
+    @Before
+    public void setUp() throws IOException {
+        temporaryFile = Files.createTempFile(fileSystem.getPath("/"), "", "");
+    }
+
+    @Test
+    public void testSuccess() throws Exception {
+        when(commandLine.getTimeout()).thenReturn(Duration.ofHours(1));
+        when(commandLine.getMaxOutputBytes()).thenReturn(10L);
+        when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8);
+        when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2));
+        when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3));
+        when(commandLine.toString()).thenReturn("program arg");
+
+        when(timer.currentTime()).thenReturn(
+                Instant.ofEpochMilli(1),
+                Instant.ofEpochMilli(2));
+
+        when(processApi.waitFor(anyLong(), any())).thenReturn(true);
+
+        try (ChildProcess2Impl child =
+                     new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) {
+            child.waitForTermination();
+        }
+    }
+
+    @Test
+    public void testTimeout() throws Exception {
+        when(commandLine.getTimeout()).thenReturn(Duration.ofSeconds(1));
+        when(commandLine.getMaxOutputBytes()).thenReturn(10L);
+        when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8);
+        when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2));
+        when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3));
+        when(commandLine.toString()).thenReturn("program arg");
+
+        when(timer.currentTime()).thenReturn(
+                Instant.ofEpochSecond(0),
+                Instant.ofEpochSecond(2));
+
+        when(processApi.waitFor(anyLong(), any())).thenReturn(true);
+
+        try (ChildProcess2Impl child =
+                     new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) {
+            try {
+                child.waitForTermination();
+                fail();
+            } catch (TimeoutChildProcessException e) {
+                assertEquals(
+                        "Command 'program arg' timed out after PT1S: stdout/stderr: ''",
+                        e.getMessage());
+            }
+        }
+    }
+
+    @Test
+    public void testMaxOutputBytes() throws Exception {
+        when(commandLine.getTimeout()).thenReturn(Duration.ofSeconds(1));
+        when(commandLine.getMaxOutputBytes()).thenReturn(10L);
+        when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8);
+        when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2));
+        when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3));
+        when(commandLine.toString()).thenReturn("program arg");
+
+        when(timer.currentTime()).thenReturn(
+                Instant.ofEpochMilli(0),
+                Instant.ofEpochMilli(1));
+
+        when(processApi.waitFor(anyLong(), any())).thenReturn(true);
+
+        Files.write(temporaryFile, "1234567890123".getBytes(StandardCharsets.UTF_8));
+
+        try (ChildProcess2Impl child =
+                     new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) {
+            try {
+                child.waitForTermination();
+                fail();
+            } catch (LargeOutputChildProcessException e) {
+                assertEquals(
+                        "Command 'program arg' output more than 13 bytes: stdout/stderr: '1234567890123'",
+                        e.getMessage());
+            }
+        }
+    }
+
+    @Test
+    public void testUnkillable() throws Exception {
+        when(commandLine.getTimeout()).thenReturn(Duration.ofSeconds(1));
+        when(commandLine.getMaxOutputBytes()).thenReturn(10L);
+        when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8);
+        when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2));
+        when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3));
+        when(commandLine.toString()).thenReturn("program arg");
+
+        when(timer.currentTime()).thenReturn(
+                Instant.ofEpochMilli(0),
+                Instant.ofEpochMilli(1));
+
+        when(processApi.waitFor(anyLong(), any())).thenReturn(false);
+
+        Files.write(temporaryFile, "1234567890123".getBytes(StandardCharsets.UTF_8));
+
+        try (ChildProcess2Impl child =
+                     new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) {
+            try {
+                child.waitForTermination();
+                fail();
+            } catch (UnkillableChildProcessException e) {
+                assertEquals(
+                        "Command 'program arg' did not terminate even after SIGTERM, +PT2M, SIGKILL, and +PT3M: stdout/stderr: '1234567890123'",
+                        e.getMessage());
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java
new file mode 100644
index 00000000000..397380461d6
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java
@@ -0,0 +1,147 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext;
+import org.junit.After;
+import org.junit.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Predicate;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class CommandLineTest {
+    private final TestTerminal terminal = new TestTerminal();
+    private final TestTaskContext context = new TestTaskContext();
+    private final CommandLine commandLine = terminal.newCommandLine(context);
+
+    @After
+    public void tearDown() {
+        terminal.verifyAllCommandsExecuted();
+    }
+
+    @Test
+    public void testStrings() {
+        terminal.expectCommand(
+                "/bin/bash \"with space\" \"speci&l\" \"\" \"double\\\"quote\" 2>&1",
+                0,
+                "");
+        commandLine.add("/bin/bash", "with space", "speci&l", "", "double\"quote").execute();
+        assertEquals("bash", commandLine.programName());
+    }
+
+    @Test
+    public void testBasicExecute() {
+        terminal.expectCommand("foo bar 2>&1", 0, "line1\nline2\n\n");
+        CommandResult result = commandLine.add("foo", "bar").execute();
+        assertEquals(0, result.getExitCode());
+        assertEquals("line1\nline2", result.getOutput());
+        assertEquals("line1\nline2\n\n", result.getUntrimmedOutput());
+        assertEquals(Arrays.asList("line1", "line2"), result.getOutputLines());
+        assertEquals(1, context.getSystemModificationLog().size());
+        assertEquals("Executing command: foo bar 2>&1", context.getSystemModificationLog().get(0));
+
+        List<CommandLine> commandLines = terminal.getTestProcessFactory().getMutableCommandLines();
+        assertEquals(1, commandLines.size());
+        assertTrue(commandLine == commandLines.get(0));
+
+        int lines = result.map(r -> r.getOutputLines().size());
+        assertEquals(2, lines);
+    }
+
+    @Test
+    public void verifyDefaults() {
+        assertEquals(CommandLine.DEFAULT_TIMEOUT, commandLine.getTimeout());
+        assertEquals(CommandLine.DEFAULT_MAX_OUTPUT_BYTES, commandLine.getMaxOutputBytes());
+        assertEquals(CommandLine.DEFAULT_SIGTERM_GRACE_PERIOD, commandLine.getSigTermGracePeriod());
+        assertEquals(CommandLine.DEFAULT_SIGKILL_GRACE_PERIOD, commandLine.getSigKillGracePeriod());
+        assertEquals(0, commandLine.getArguments().size());
+        assertEquals(StandardCharsets.UTF_8, commandLine.getOutputEncoding());
+        assertTrue(commandLine.getRedirectStderrToStdoutInsteadOfDiscard());
+        Predicate<Integer> defaultExitCodePredicate = commandLine.getSuccessfulExitCodePredicate();
+        assertTrue(defaultExitCodePredicate.test(0));
+        assertFalse(defaultExitCodePredicate.test(1));
+    }
+
+    @Test
+    public void executeSilently() {
+        terminal.ignoreCommand("");
+        commandLine.add("foo", "bar").executeSilently();
+        assertEquals(0, context.getSystemModificationLog().size());
+        commandLine.recordSilentExecutionAsSystemModification();
+        assertEquals(1, context.getSystemModificationLog().size());
+        assertEquals("Executed command: foo bar 2>&1", context.getSystemModificationLog().get(0));
+    }
+
+    @Test(expected = NegativeArraySizeException.class)
+    public void processFactorySpawnFails() {
+        terminal.interceptCommand(
+                        commandLine.toString(),
+                        command -> { throw new NegativeArraySizeException(); });
+        commandLine.add("foo").execute();
+    }
+
+    @Test
+    public void waitingForTerminationExceptionStillClosesChild() {
+        TestChildProcess2 child = new TestChildProcess2(0, "");
+        child.throwInWaitForTermination(new NegativeArraySizeException());
+        terminal.interceptCommand(commandLine.toString(), command -> child);
+        assertFalse(child.closeCalled());
+        try {
+            commandLine.add("foo").execute();
+            fail();
+        } catch (NegativeArraySizeException e) {
+            // OK
+        }
+
+        assertTrue(child.closeCalled());
+    }
+
+    @Test
+    public void programFails() {
+        TestChildProcess2 child = new TestChildProcess2(0, "");
+        terminal.expectCommand("foo 2>&1", 1, "");
+        try {
+            commandLine.add("foo").execute();
+            fail();
+        } catch (ChildProcessFailureException e) {
+            assertEquals(
+                    "Command 'foo 2>&1' terminated with exit code 1: stdout/stderr: ''",
+                    e.getMessage());
+        }
+    }
+
+    @Test
+    public void mapException() {
+        terminal.ignoreCommand("output");
+        CommandResult result = terminal.newCommandLine(context).add("program").execute();
+        IllegalArgumentException exception = new IllegalArgumentException("foo");
+        try {
+            result.mapOutput(output -> { throw exception; });
+            fail();
+        } catch (UnexpectedOutputException2 e) {
+            assertEquals("Command 'program 2>&1' output was not of the expected format: " +
+                    "Failed to map output: stdout/stderr: 'output'", e.getMessage());
+            assertTrue(e.getCause() == exception);
+        }
+    }
+
+    @Test
+    public void testMapEachLine() {
+        assertEquals(
+                1 + 2 + 3,
+                terminal.ignoreCommand("1\n2\n3\n")
+                        .newCommandLine(context)
+                        .add("foo")
+                        .execute()
+                        .mapEachLine(Integer::valueOf)
+                        .stream()
+                        .mapToInt(i -> i)
+                        .sum());
+    }
+}
\ No newline at end of file
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java
new file mode 100644
index 00000000000..5a32b4c68b1
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java
@@ -0,0 +1,48 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+import com.yahoo.vespa.hosted.node.admin.task.util.time.TestTimer;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class ProcessFactoryImplTest {
+    private final ProcessStarter starter = mock(ProcessStarter.class);
+    private final TestTimer timer = new TestTimer();
+    private final ProcessFactoryImpl processFactory = new ProcessFactoryImpl(starter, timer);
+
+    @Test
+    public void testSpawn() {
+        CommandLine commandLine = mock(CommandLine.class);
+        when(commandLine.getArguments()).thenReturn(Arrays.asList("program"));
+        when(commandLine.getRedirectStderrToStdoutInsteadOfDiscard()).thenReturn(true);
+        when(commandLine.programName()).thenReturn("program");
+        Path outputPath;
+        try (ChildProcess2Impl child = processFactory.spawn(commandLine)) {
+            outputPath = child.getOutputPath();
+            assertTrue(Files.exists(outputPath));
+            assertEquals("rw-------", new UnixPath(outputPath).getPermissions());
+            ArgumentCaptor<ProcessBuilder> processBuilderCaptor =
+                    ArgumentCaptor.forClass(ProcessBuilder.class);
+            verify(starter).start(processBuilderCaptor.capture());
+            ProcessBuilder processBuilder = processBuilderCaptor.getValue();
+            assertTrue(processBuilder.redirectErrorStream());
+            ProcessBuilder.Redirect redirect = processBuilder.redirectOutput();
+            assertEquals(ProcessBuilder.Redirect.Type.WRITE, redirect.type());
+            assertEquals(outputPath.toFile(), redirect.file());
+        }
+
+        assertFalse(Files.exists(outputPath));
+    }
+}
\ No newline at end of file
diff --git a/searchcore/src/apps/proton/proton.cpp b/searchcore/src/apps/proton/proton.cpp
index 5cdc8e04eb4..70cd8d8a7ad 100644
--- a/searchcore/src/apps/proton/proton.cpp
+++ b/searchcore/src/apps/proton/proton.cpp
@@ -208,6 +208,11 @@ App::Main()
             }
             sigBusHandler.reset(new search::SigBusHandler(stateFile.get()));
             ioErrorHandler.reset(new search::IOErrorHandler(stateFile.get()));
+            if ( ! params.serviceidentity.empty()) {
+                proton.getMetricManager().init(params.serviceidentity, proton.getThreadPool());
+            } else {
+                proton.getMetricManager().init(params.identity, proton.getThreadPool());
+            }
             if (!downPersistence) {
                 proton.init(configSnapshot);
             }
@@ -218,8 +223,6 @@ App::Main()
                 spiProton->setupConfig(params.subscribeTimeout);
                 spiProton->createNode();
                 EV_STARTED("servicelayer");
-            } else {
-                proton.getMetricManager().init(params.identity, proton.getThreadPool());
             }
             EV_STARTED("proton");
             while (!(SIG::INT.check() || SIG::TERM.check() || (spiProton && spiProton->getNode().attemptedStopped()))) {
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
index 65a47e1ded8..49f8d50d9d4 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
@@ -165,7 +165,7 @@ Proton::Proton(const config::ConfigUri & configUri,
       _configUri(configUri),
       _mutex(),
       _metricsHook(*this),
-      _metricsEngine(),
+      _metricsEngine(std::make_unique<MetricsEngine>()),
       _fileHeaderContext(*this, progName),
       _tls(),
       _diskMemUsageSampler(),
@@ -235,7 +235,6 @@ Proton::init(const BootstrapConfig::SP & configSnapshot)
                            (protonConfig.basedir,
                             diskMemUsageSamplerConfig(protonConfig, hwInfo));
 
-    _metricsEngine.reset(new MetricsEngine());
     _metricsEngine->addMetricsHook(_metricsHook);
     _fileHeaderContext.setClusterName(protonConfig.clustername, protonConfig.basedir);
     _tls.reset(new TLS(_configUri.createWithNewId(protonConfig.tlsconfigid), _fileHeaderContext));
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/OperationMapper.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/OperationMapper.java
index b8f8e288257..55782c36d18 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/OperationMapper.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/OperationMapper.java
@@ -219,7 +219,7 @@ class OperationMapper {
     private static Optional<TypedTensorFunction> placeholderWithDefault(TensorFlowImporter.Parameters params) {
         String name = toVespaName(params.node().getInput(0));
         Tensor defaultValue = getConstantTensor(params, params.node().getInput(0));
-        params.result().constant(name, defaultValue);
+        params.result().largeConstant(name, defaultValue);
         params.result().macro(name, new RankingExpression(name, new ReferenceNode("constant(\"" + name + "\")")));
         // The default value will be provided by the macro. Users can override macro to change value.
         TypedTensorFunction output = new TypedTensorFunction(defaultValue.type(), new VariableTensor(name));
@@ -544,7 +544,11 @@ class OperationMapper {
 
     private static Optional<TypedTensorFunction> createConstant(TensorFlowImporter.Parameters params, Tensor constant) {
         String name = toVespaName(params.node().getName());
-        params.result().constant(name, constant);
+        if (constant.type().rank() == 0 || constant.size() <= 1) {
+            params.result().smallConstant(name, constant);
+        } else {
+            params.result().largeConstant(name, constant);
+        }
         TypedTensorFunction output = new TypedTensorFunction(constant.type(),
                 new TensorFunctionNode.TensorFunctionExpressionNode(
                         new ReferenceNode("constant(\"" + name + "\")")));
@@ -553,8 +557,11 @@ class OperationMapper {
 
     private static Tensor getConstantTensor(TensorFlowImporter.Parameters params, String name) {
         String vespaName = toVespaName(name);
-        if (params.result().constants().containsKey(vespaName)) {
-            return params.result().constants().get(vespaName);
+        if (params.result().smallConstants().containsKey(vespaName)) {
+            return params.result().smallConstants().get(vespaName);
+        }
+        if (params.result().largeConstants().containsKey(vespaName)) {
+            return params.result().largeConstants().get(vespaName);
         }
         Session.Runner fetched = params.model().session().runner().fetch(name);
         List<org.tensorflow.Tensor<?>> importedTensors = fetched.run();
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowModel.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowModel.java
index 530f4793b62..351aa417f9c 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowModel.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowModel.java
@@ -24,13 +24,15 @@ public class TensorFlowModel {
 
     private final Map<String, Signature> signatures = new HashMap<>();
     private final Map<String, TensorType> arguments = new HashMap<>();
-    private final Map<String, Tensor> constants = new HashMap<>();
+    private final Map<String, Tensor> smallConstants = new HashMap<>();
+    private final Map<String, Tensor> largeConstants = new HashMap<>();
     private final Map<String, RankingExpression> expressions = new HashMap<>();
     private final Map<String, RankingExpression> macros = new HashMap<>();
     private final Map<String, TensorType> requiredMacros = new HashMap<>();
 
     void argument(String name, TensorType argumentType) { arguments.put(name, argumentType); }
-    void constant(String name, Tensor constant) { constants.put(name, constant); }
+    void smallConstant(String name, Tensor constant) { smallConstants.put(name, constant); }
+    void largeConstant(String name, Tensor constant) { largeConstants.put(name, constant); }
     void expression(String name, RankingExpression expression) { expressions.put(name, expression); }
     void macro(String name, RankingExpression expression) { macros.put(name, expression); }
     void requiredMacro(String name, TensorType type) { requiredMacros.put(name, type); }
@@ -43,8 +45,19 @@ public class TensorFlowModel {
     /** Returns an immutable map of the arguments ("Placeholders") of this */
     public Map<String, TensorType> arguments() { return Collections.unmodifiableMap(arguments); }
 
-    /** Returns an immutable map of the constants of this */
-    public Map<String, Tensor> constants() { return Collections.unmodifiableMap(constants); }
+    /**
+     * Returns an immutable map of the small constants of this.
+     * These should have sizes up to a few kb at most, and correspond to constant
+     * values given in the TensorFlow source.
+     */
+    public Map<String, Tensor> smallConstants() { return Collections.unmodifiableMap(smallConstants); }
+
+    /**
+     * Returns an immutable map of the large constants of this.
+     * These can have sizes in gigabytes and must be distributed to nodes separately from configuration,
+     * and correspond to Variable files stored separately in TensorFlow.
+     */
+    public Map<String, Tensor> largeConstants() { return Collections.unmodifiableMap(largeConstants); }
 
     /**
      * Returns an immutable map of the expressions of this - corresponding to TensorFlow nodes
diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/MnistSoftmaxImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/MnistSoftmaxImportTestCase.java
index 01dd15d5fa0..ad5abd4c03d 100644
--- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/MnistSoftmaxImportTestCase.java
+++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/MnistSoftmaxImportTestCase.java
@@ -20,15 +20,15 @@ public class MnistSoftmaxImportTestCase {
         TestableTensorFlowModel model = new TestableTensorFlowModel("src/test/files/integration/tensorflow/mnist_softmax/saved");
 
         // Check constants
-        assertEquals(2, model.get().constants().size());
+        assertEquals(2, model.get().largeConstants().size());
 
-        Tensor constant0 = model.get().constants().get("Variable");
+        Tensor constant0 = model.get().largeConstants().get("Variable");
         assertNotNull(constant0);
         assertEquals(new TensorType.Builder().indexed("d0", 784).indexed("d1", 10).build(),
                      constant0.type());
         assertEquals(7840, constant0.size());
 
-        Tensor constant1 = model.get().constants().get("Variable_1");
+        Tensor constant1 = model.get().largeConstants().get("Variable_1");
         assertNotNull(constant1);
         assertEquals(new TensorType.Builder().indexed("d0", 10).build(),
                      constant1.type());
diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TestableTensorFlowModel.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TestableTensorFlowModel.java
index 2c621fd2e92..ae7714b271a 100644
--- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TestableTensorFlowModel.java
+++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TestableTensorFlowModel.java
@@ -57,7 +57,8 @@ public class TestableTensorFlowModel {
 
     private Context contextFrom(TensorFlowModel result) {
         MapContext context = new MapContext();
-        result.constants().forEach((name, tensor) -> context.put("constant(\"" + name + "\")", new TensorValue(tensor)));
+        result.largeConstants().forEach((name, tensor) -> context.put("constant(\"" + name + "\")", new TensorValue(tensor)));
+        result.smallConstants().forEach((name, tensor) -> context.put("constant(\"" + name + "\")", new TensorValue(tensor)));
         return context;
     }
 
diff --git a/storage/src/tests/storageserver/configurable_bucket_resolver_test.cpp b/storage/src/tests/storageserver/configurable_bucket_resolver_test.cpp
index 4d80192d9e9..c10db0a1acd 100644
--- a/storage/src/tests/storageserver/configurable_bucket_resolver_test.cpp
+++ b/storage/src/tests/storageserver/configurable_bucket_resolver_test.cpp
@@ -20,6 +20,7 @@ struct ConfigurableBucketResolverTest : CppUnit::TestFixture {
     CPPUNIT_TEST(known_bucket_space_is_resolved_from_document_id);
     CPPUNIT_TEST(unknown_bucket_space_in_id_throws_exception);
     CPPUNIT_TEST(can_create_resolver_from_bucket_space_config);
+    CPPUNIT_TEST(legacy_document_id_without_document_type_maps_to_default_space);
     CPPUNIT_TEST_SUITE_END();
 
     using BucketSpaceMapping = ConfigurableBucketResolver::BucketSpaceMapping;
@@ -47,6 +48,7 @@ struct ConfigurableBucketResolverTest : CppUnit::TestFixture {
     void known_bucket_space_is_resolved_from_document_id();
     void unknown_bucket_space_in_id_throws_exception();
     void can_create_resolver_from_bucket_space_config();
+    void legacy_document_id_without_document_type_maps_to_default_space();
 };
 
 CPPUNIT_TEST_SUITE_REGISTRATION(ConfigurableBucketResolverTest);
@@ -133,5 +135,11 @@ void ConfigurableBucketResolverTest::can_create_resolver_from_bucket_space_confi
                          resolver->bucketFromId(DocumentId("id::baz::xyz")).getBucketSpace());
 }
 
+void ConfigurableBucketResolverTest::legacy_document_id_without_document_type_maps_to_default_space() {
+    auto resolver = create_simple_resolver();
+    CPPUNIT_ASSERT_EQUAL(document::FixedBucketSpaces::default_space(),
+                         resolver.bucketFromId(DocumentId("userdoc:baz:1234:baz")).getBucketSpace());
+}
+
 }
 
diff --git a/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.cpp b/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.cpp
index bd4a60f851a..79e32551560 100644
--- a/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.cpp
+++ b/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.cpp
@@ -5,30 +5,36 @@
 #include <vespa/vespalib/util/exceptions.h>
 #include "configurable_bucket_resolver.h"
 
+using namespace document;
+
 namespace storage {
 
-document::Bucket ConfigurableBucketResolver::bucketFromId(const document::DocumentId& id) const {
+document::Bucket ConfigurableBucketResolver::bucketFromId(const DocumentId& id) const {
+    if (!id.hasDocType()) {
+        // Legacy document ids without document type maps to default bucket space
+        return Bucket(FixedBucketSpaces::default_space(), BucketId(0));
+    }
     auto iter = _type_to_space.find(id.getDocType());
     if (iter != _type_to_space.end()) {
-        return document::Bucket(iter->second, document::BucketId(0));
+        return Bucket(iter->second, BucketId(0));
     }
-    throw document::UnknownBucketSpaceException("Unknown bucket space mapping for document type '"
-                                           + id.getDocType() + "' in id: '" + id.toString() + "'", VESPA_STRLOC);
+    throw UnknownBucketSpaceException("Unknown bucket space mapping for document type '"
+                                      + id.getDocType() + "' in id: '" + id.toString() + "'", VESPA_STRLOC);
 }
 
-document::BucketSpace ConfigurableBucketResolver::bucketSpaceFromName(const vespalib::string& name) const {
-    return document::FixedBucketSpaces::from_string(name);
+BucketSpace ConfigurableBucketResolver::bucketSpaceFromName(const vespalib::string& name) const {
+    return FixedBucketSpaces::from_string(name);
 }
 
-vespalib::string ConfigurableBucketResolver::nameFromBucketSpace(const document::BucketSpace& space) const {
-    return document::FixedBucketSpaces::to_string(space);
+vespalib::string ConfigurableBucketResolver::nameFromBucketSpace(const BucketSpace& space) const {
+    return FixedBucketSpaces::to_string(space);
 }
 
 std::shared_ptr<ConfigurableBucketResolver> ConfigurableBucketResolver::from_config(
         const vespa::config::content::core::BucketspacesConfig& config) {
     ConfigurableBucketResolver::BucketSpaceMapping type_to_space;
     for (auto& mapping : config.documenttype) {
-        type_to_space.emplace(mapping.name, document::FixedBucketSpaces::from_string(mapping.bucketspace));
+        type_to_space.emplace(mapping.name, FixedBucketSpaces::from_string(mapping.bucketspace));
     }
     return std::make_shared<ConfigurableBucketResolver>(std::move(type_to_space));
 }
diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp
index 6be8f0a5ec8..aa5475df823 100644
--- a/storage/src/vespa/storage/storageserver/storagenode.cpp
+++ b/storage/src/vespa/storage/storageserver/storagenode.cpp
@@ -215,7 +215,9 @@ StorageNode::initialize()
     // and the like. Note that at this time, all metrics should hopefully
     // have been created, such that we don't need to pay the extra cost of
     // reinitializing metric manager often.
-    _context.getComponentRegister().getMetricManager().init(_configUri, _context.getThreadPool());
+    if ( ! _context.getComponentRegister().getMetricManager().isInitialized() ) {
+        _context.getComponentRegister().getMetricManager().init(_configUri, _context.getThreadPool());
+    }
 
     if (_chain) {
         LOG(debug, "Storage chain configured. Calling open()");