Merge pull request #19446 from vespa-engine/mpolden/vespa-cli-prod

vespa prod

15 files changed, 1361 insertions, 67 deletions

diff --git a/client/go/cmd/command_tester.go b/client/go/cmd/command_tester.go
index 8eaf6be2c22..2d2de6a201c 100644
--- a/client/go/cmd/command_tester.go
+++ b/client/go/cmd/command_tester.go
@@ -7,6 +7,7 @@ package cmd
 import (
 	"bytes"
 	"crypto/tls"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -23,6 +24,7 @@ import (
 type command struct {
 	homeDir  string
 	cacheDir string
+	stdin    io.ReadWriter
 	args     []string
 	moreArgs []string
 }
@@ -56,6 +58,8 @@ func execute(cmd command, t *testing.T, client *mockHttpClient) (string, string)
 	os.Setenv("VESPA_CLI_CACHE_DIR", cmd.cacheDir)
 
 	// Reset flags to their default value - persistent flags in Cobra persists over tests
+	// TODO: Due to the bad design of viper, the only proper fix is to get rid of global state by moving each command to
+	// their own sub-package
 	rootCmd.Flags().VisitAll(resetFlag)
 	documentCmd.Flags().VisitAll(resetFlag)
 
@@ -67,6 +71,11 @@ func execute(cmd command, t *testing.T, client *mockHttpClient) (string, string)
 	var capturedErr bytes.Buffer
 	stdout = &capturedOut
 	stderr = &capturedErr
+	if cmd.stdin != nil {
+		stdin = cmd.stdin
+	} else {
+		stdin = os.Stdin
+	}
 
 	// Execute command and return output
 	rootCmd.SetArgs(append(cmd.args, cmd.moreArgs...))
diff --git a/client/go/cmd/deploy.go b/client/go/cmd/deploy.go
index 1380eca5bbb..034dac2e67b 100644
--- a/client/go/cmd/deploy.go
+++ b/client/go/cmd/deploy.go
@@ -44,7 +44,10 @@ If application directory is not specified, it defaults to working directory.
 When deploying to Vespa Cloud the system can be overridden by setting the
 environment variable VESPA_CLI_CLOUD_SYSTEM. This is intended for internal use
 only.`,
-	Example:           "$ vespa deploy .",
+	Example: `$ vespa deploy .
+$ vespa deploy -t cloud
+$ vespa deploy -t cloud -z dev.aws-us-east-1c  # -z can be omitted here as this zone is the default
+$ vespa deploy -t cloud -z perf.aws-us-east-1c`,
 	Args:              cobra.MaximumNArgs(1),
 	DisableAutoGenTag: true,
 	Run: func(cmd *cobra.Command, args []string) {
@@ -59,20 +62,7 @@ only.`,
 			return
 		}
 		target := getTarget()
-		opts := vespa.DeploymentOpts{ApplicationPackage: pkg, Target: target}
-		if opts.IsCloud() {
-			deployment := deploymentFromArgs()
-			if !opts.ApplicationPackage.HasCertificate() {
-				fatalErrHint(fmt.Errorf("Missing certificate in application package"), "Applications in Vespa Cloud require a certificate", "Try 'vespa cert'")
-				return
-			}
-			opts.APIKey, err = cfg.ReadAPIKey(deployment.Application.Tenant)
-			if err != nil {
-				fatalErrHint(err, "Deployment to cloud requires an API key. Try 'vespa api-key'")
-				return
-			}
-			opts.Deployment = deployment
-		}
+		opts := getDeploymentOpts(cfg, pkg, target)
 		if sessionOrRunID, err := vespa.Deploy(opts); err == nil {
 			if opts.IsCloud() {
 				printSuccess("Triggered deployment of ", color.Cyan(pkg.Path), " with run ID ", color.Cyan(sessionOrRunID))
diff --git a/client/go/cmd/helpers.go b/client/go/cmd/helpers.go
index b5525cf11fe..4768290e33e 100644
--- a/client/go/cmd/helpers.go
+++ b/client/go/cmd/helpers.go
@@ -30,7 +30,9 @@ func fatalErr(err error, msg ...interface{}) {
 }
 
 func printErrHint(err error, hints ...string) {
-	printErr(nil, err.Error())
+	if err != nil {
+		printErr(nil, err.Error())
+	}
 	for _, hint := range hints {
 		fmt.Fprintln(stderr, color.Cyan("Hint:"), hint)
 	}
@@ -140,9 +142,10 @@ func getService(service string, sessionOrRunID int64) *vespa.Service {
 	return s
 }
 
+func getSystem() string { return os.Getenv("VESPA_CLI_CLOUD_SYSTEM") }
+
 func getConsoleURL() string {
-	system := os.Getenv("VESPA_CLI_CLOUD_SYSTEM")
-	if system == "publiccd" {
+	if getSystem() == "publiccd" {
 		return "https://console-cd.vespa.oath.cloud"
 	}
 	return "https://console.vespa.oath.cloud"
@@ -150,8 +153,7 @@ func getConsoleURL() string {
 }
 
 func getApiURL() string {
-	system := os.Getenv("VESPA_CLI_CLOUD_SYSTEM")
-	if system == "publiccd" {
+	if getSystem() == "publiccd" {
 		return "https://api.vespa-external-cd.aws.oath.cloud:4443"
 	}
 	return "https://api.vespa-external.aws.oath.cloud:4443"
@@ -221,3 +223,22 @@ func waitForService(service string, sessionOrRunID int64) {
 		fatalErr(err, s.Description(), " at ", color.Cyan(s.BaseURL), " is ", color.Red("not ready"))
 	}
 }
+
+func getDeploymentOpts(cfg *Config, pkg vespa.ApplicationPackage, target vespa.Target) vespa.DeploymentOpts {
+	opts := vespa.DeploymentOpts{ApplicationPackage: pkg, Target: target}
+	if opts.IsCloud() {
+		deployment := deploymentFromArgs()
+		if !opts.ApplicationPackage.HasCertificate() {
+			fatalErrHint(fmt.Errorf("Missing certificate in application package"), "Applications in Vespa Cloud require a certificate", "Try 'vespa cert'")
+			return opts
+		}
+		var err error
+		opts.APIKey, err = cfg.ReadAPIKey(deployment.Application.Tenant)
+		if err != nil {
+			fatalErrHint(err, "Deployment to cloud requires an API key. Try 'vespa api-key'")
+			return opts
+		}
+		opts.Deployment = deployment
+	}
+	return opts
+}
diff --git a/client/go/cmd/prod.go b/client/go/cmd/prod.go
new file mode 100644
index 00000000000..382ede0fae8
--- /dev/null
+++ b/client/go/cmd/prod.go
@@ -0,0 +1,349 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package cmd
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"github.com/vespa-engine/vespa/client/go/util"
+	"github.com/vespa-engine/vespa/client/go/vespa"
+	"github.com/vespa-engine/vespa/client/go/vespa/xml"
+)
+
+func init() {
+	rootCmd.AddCommand(prodCmd)
+	prodCmd.AddCommand(prodInitCmd)
+	prodCmd.AddCommand(prodSubmitCmd)
+}
+
+var prodCmd = &cobra.Command{
+	Use:   "prod",
+	Short: "Deploy an application package to production in Vespa Cloud",
+	Long: `Deploy an application package to production in Vespa Cloud.
+
+Configure and deploy your application package to production in Vespa Cloud.`,
+	Example: `$ vespa prod init
+$ vespa prod submit`,
+	DisableAutoGenTag: true,
+	Run: func(cmd *cobra.Command, args []string) {
+		// Root command does nothing
+		cmd.Help()
+		exitFunc(1)
+	},
+}
+
+var prodInitCmd = &cobra.Command{
+	Use:   "init",
+	Short: "Modify service.xml and deployment.xml for production deployment",
+	Long: `Modify service.xml and deployment.xml for production deployment.
+
+Only basic deployment configuration is available through this command. For
+advanced configuration see the relevant Vespa Cloud documentation and make
+changes to deployment.xml and services.xml directly.
+
+Reference:
+https://cloud.vespa.ai/en/reference/services
+https://cloud.vespa.ai/en/reference/deployment`,
+	DisableAutoGenTag: true,
+	Run: func(cmd *cobra.Command, args []string) {
+		appSource := applicationSource(args)
+		pkg, err := vespa.FindApplicationPackage(appSource, false)
+		if err != nil {
+			fatalErr(err)
+			return
+		}
+		if pkg.IsZip() {
+			fatalErrHint(fmt.Errorf("Cannot modify compressed application package %s", pkg.Path),
+				"Try running 'mvn clean' and run this command again")
+			return
+		}
+
+		deploymentXML, err := readDeploymentXML(pkg)
+		if err != nil {
+			fatalErr(err, "Could not read deployment.xml")
+			return
+		}
+		servicesXML, err := readServicesXML(pkg)
+		if err != nil {
+			fatalErr(err, "A services.xml declaring your cluster(s) must exist")
+			return
+		}
+
+		fmt.Fprint(stdout, "This will modify any existing ", color.Yellow("deployment.xml"), " and ", color.Yellow("services.xml"),
+			"!\nBefore modification a backup of the original file will be created.\n\n")
+		fmt.Fprint(stdout, "A default value is suggested (shown inside brackets) based on\nthe files' existing contents. Press enter to use it.\n\n")
+		fmt.Fprint(stdout, "Abort the configuration at any time by pressing Ctrl-C. The\nfiles will remain untouched.\n\n")
+		r := bufio.NewReader(stdin)
+		deploymentXML = updateRegions(r, deploymentXML)
+		servicesXML = updateNodes(r, servicesXML)
+
+		fmt.Fprintln(stdout)
+		if err := writeWithBackup(pkg, "deployment.xml", deploymentXML.String()); err != nil {
+			fatalErr(err)
+			return
+		}
+		if err := writeWithBackup(pkg, "services.xml", servicesXML.String()); err != nil {
+			fatalErr(err)
+			return
+		}
+	},
+}
+
+var prodSubmitCmd = &cobra.Command{
+	Use:   "submit",
+	Short: "Submit your application for production deployment",
+	Long: `Submit your application for production deployment.
+
+This commands uploads your application package to Vespa Cloud and deploys it to
+the production zones specified in deployment.xml.
+
+Nodes are allocated to your application according to resources specified in
+services.xml.
+
+While submitting an application from a local development environment is
+supported, it's strongly recommended that production deployments are performed
+by a continuous build system.
+
+For more information about production deployments in Vespa Cloud see:
+https://cloud.vespa.ai/en/getting-to-production
+https://cloud.vespa.ai/en/automated-deployments`,
+	DisableAutoGenTag: true,
+	Example: `$ mvn package
+$ vespa prod submit`,
+	Run: func(cmd *cobra.Command, args []string) {
+		target := getTarget()
+		if target.Type() != "cloud" {
+			fatalErr(fmt.Errorf("%s target cannot deploy to Vespa Cloud", target.Type()))
+			return
+		}
+		appSource := applicationSource(args)
+		pkg, err := vespa.FindApplicationPackage(appSource, true)
+		if err != nil {
+			fatalErr(err)
+			return
+		}
+		cfg, err := LoadConfig()
+		if err != nil {
+			fatalErr(err, "Could not load config")
+			return
+		}
+		if !pkg.HasDeployment() {
+			fatalErrHint(fmt.Errorf("No deployment.xml found"), "Try creating one with vespa prod init")
+			return
+		}
+		if !pkg.IsJava() {
+			// TODO: Loosen this requirement when we start supporting applications with Java in production
+			fatalErrHint(fmt.Errorf("No jar files found in %s", pkg.Path), "Only applications containing Java components are currently supported")
+			return
+		}
+		isCI := os.Getenv("CI") != ""
+		if !isCI {
+			fmt.Fprintln(stderr, color.Yellow("Warning:"), "Submitting from a non-CI environment is discouraged")
+			printErrHint(nil, "See https://cloud.vespa.ai/en/getting-to-production for best practices")
+		}
+		opts := getDeploymentOpts(cfg, pkg, target)
+		if err := vespa.Submit(opts); err != nil {
+			fatalErr(err, "Could not submit application for deployment")
+		} else {
+			printSuccess("Submitted ", color.Cyan(pkg.Path), " for deployment")
+			log.Printf("See %s for deployment progress\n", color.Cyan(fmt.Sprintf("%s/tenant/%s/application/%s/prod/deployment",
+				getConsoleURL(), opts.Deployment.Application.Tenant, opts.Deployment.Application.Application)))
+		}
+	},
+}
+
+func writeWithBackup(pkg vespa.ApplicationPackage, filename, contents string) error {
+	dst := filepath.Join(pkg.Path, filename)
+	if util.PathExists(dst) {
+		data, err := ioutil.ReadFile(dst)
+		if err != nil {
+			return err
+		}
+		if bytes.Equal(data, []byte(contents)) {
+			fmt.Fprintf(stdout, "Not writing %s: File is unchanged\n", color.Yellow(filename))
+			return nil
+		}
+		renamed := false
+		for i := 1; i <= 1000; i++ {
+			bak := fmt.Sprintf("%s.%d.bak", dst, i)
+			if !util.PathExists(bak) {
+				fmt.Fprintf(stdout, "Backing up existing %s to %s\n", color.Yellow(filename), color.Yellow(bak))
+				if err := os.Rename(dst, bak); err != nil {
+					return err
+				}
+				renamed = true
+				break
+			}
+		}
+		if !renamed {
+			return fmt.Errorf("could not find an unused backup name for %s", dst)
+		}
+	}
+	fmt.Fprintf(stdout, "Writing %s\n", color.Green(dst))
+	return ioutil.WriteFile(dst, []byte(contents), 0644)
+}
+
+func updateRegions(r *bufio.Reader, deploymentXML xml.Deployment) xml.Deployment {
+	regions := promptRegions(r, deploymentXML)
+	parts := strings.Split(regions, ",")
+	regionElements := xml.Regions(parts...)
+	if err := deploymentXML.Replace("prod", "region", regionElements); err != nil {
+		fatalErr(err, "Could not update region elements in deployment.xml")
+	}
+	// TODO: Some sample apps come with production <test> elements, but not necessarily working production tests, we
+	//       therefore remove <test> elements here.
+	//       This can be improved by supporting <test> elements in xml package and allow specifying testing as part of
+	//       region prompt, e.g. region1;test,region2
+	if err := deploymentXML.Replace("prod", "test", nil); err != nil {
+		fatalErr(err, "Could not remove test elements in deployment.xml")
+	}
+	return deploymentXML
+}
+
+func promptRegions(r *bufio.Reader, deploymentXML xml.Deployment) string {
+	fmt.Fprintln(stdout, color.Cyan("> Deployment regions"))
+	fmt.Fprintf(stdout, "Documentation: %s\n", color.Green("https://cloud.vespa.ai/en/reference/zones"))
+	fmt.Fprintf(stdout, "Example: %s\n\n", color.Yellow("aws-us-east-1c,aws-us-west-2a"))
+	var currentRegions []string
+	for _, r := range deploymentXML.Prod.Regions {
+		currentRegions = append(currentRegions, r.Name)
+	}
+	if len(deploymentXML.Instance) > 0 {
+		for _, r := range deploymentXML.Instance[0].Prod.Regions {
+			currentRegions = append(currentRegions, r.Name)
+		}
+	}
+	validator := func(input string) error {
+		regions := strings.Split(input, ",")
+		for _, r := range regions {
+			if !xml.IsProdRegion(r, getSystem()) {
+				return fmt.Errorf("invalid region %s", r)
+			}
+		}
+		return nil
+	}
+	return prompt(r, "Which regions do you wish to deploy in?", strings.Join(currentRegions, ","), validator)
+}
+
+func updateNodes(r *bufio.Reader, servicesXML xml.Services) xml.Services {
+	for _, c := range servicesXML.Container {
+		nodes := promptNodes(r, c.ID, c.Nodes)
+		if err := servicesXML.Replace("container#"+c.ID, "nodes", nodes); err != nil {
+			fatalErr(err)
+			return xml.Services{}
+		}
+	}
+	for _, c := range servicesXML.Content {
+		nodes := promptNodes(r, c.ID, c.Nodes)
+		if err := servicesXML.Replace("content#"+c.ID, "nodes", nodes); err != nil {
+			fatalErr(err)
+			return xml.Services{}
+		}
+	}
+	return servicesXML
+}
+
+func promptNodes(r *bufio.Reader, clusterID string, defaultValue xml.Nodes) xml.Nodes {
+	count := promptNodeCount(r, clusterID, defaultValue.Count)
+	const autoSpec = "auto"
+	defaultSpec := autoSpec
+	resources := defaultValue.Resources
+	if resources != nil {
+		defaultSpec = defaultValue.Resources.String()
+	}
+	spec := promptResources(r, clusterID, defaultSpec)
+	if spec == autoSpec {
+		resources = nil
+	} else {
+		r, err := xml.ParseResources(spec)
+		if err != nil {
+			fatalErr(err) // Should not happen as resources have already been validated
+			return xml.Nodes{}
+		}
+		resources = &r
+	}
+	return xml.Nodes{Count: count, Resources: resources}
+}
+
+func promptNodeCount(r *bufio.Reader, clusterID string, nodeCount string) string {
+	fmt.Fprintln(stdout, color.Cyan("\n> Node count: "+clusterID+" cluster"))
+	fmt.Fprintf(stdout, "Documentation: %s\n", color.Green("https://cloud.vespa.ai/en/reference/services"))
+	fmt.Fprintf(stdout, "Example: %s\nExample: %s\n\n", color.Yellow("4"), color.Yellow("[2,8]"))
+	validator := func(input string) error {
+		_, _, err := xml.ParseNodeCount(input)
+		return err
+	}
+	return prompt(r, fmt.Sprintf("How many nodes should the %s cluster have?", color.Cyan(clusterID)), nodeCount, validator)
+}
+
+func promptResources(r *bufio.Reader, clusterID string, resources string) string {
+	fmt.Fprintln(stdout, color.Cyan("\n> Node resources: "+clusterID+" cluster"))
+	fmt.Fprintf(stdout, "Documentation: %s\n", color.Green("https://cloud.vespa.ai/en/reference/services"))
+	fmt.Fprintf(stdout, "Example: %s\nExample: %s\n\n", color.Yellow("auto"), color.Yellow("vcpu=4,memory=8Gb,disk=100Gb"))
+	validator := func(input string) error {
+		if input == "auto" {
+			return nil
+		}
+		_, err := xml.ParseResources(input)
+		return err
+	}
+	return prompt(r, fmt.Sprintf("Which resources should each node in the %s cluster have?", color.Cyan(clusterID)), resources, validator)
+}
+
+func readDeploymentXML(pkg vespa.ApplicationPackage) (xml.Deployment, error) {
+	f, err := os.Open(filepath.Join(pkg.Path, "deployment.xml"))
+	if errors.Is(err, os.ErrNotExist) {
+		// Return a default value if there is no current deployment.xml
+		return xml.DefaultDeployment, nil
+	} else if err != nil {
+		return xml.Deployment{}, err
+	}
+	defer f.Close()
+	return xml.ReadDeployment(f)
+}
+
+func readServicesXML(pkg vespa.ApplicationPackage) (xml.Services, error) {
+	f, err := os.Open(filepath.Join(pkg.Path, "services.xml"))
+	if err != nil {
+		return xml.Services{}, err
+	}
+	defer f.Close()
+	return xml.ReadServices(f)
+}
+
+func prompt(r *bufio.Reader, question, defaultAnswer string, validator func(input string) error) string {
+	var input string
+	for input == "" {
+		fmt.Fprint(stdout, question)
+		if defaultAnswer != "" {
+			fmt.Fprint(stdout, " [", color.Yellow(defaultAnswer), "]")
+		}
+		fmt.Fprint(stdout, " ")
+
+		var err error
+		input, err = r.ReadString('\n')
+		if err != nil {
+			fatalErr(err)
+			return ""
+		}
+		input = strings.TrimSpace(input)
+		if input == "" {
+			input = defaultAnswer
+		}
+
+		if err := validator(input); err != nil {
+			printErr(err)
+			fmt.Fprintln(stderr)
+			input = ""
+		}
+	}
+	return input
+}
diff --git a/client/go/cmd/prod_test.go b/client/go/cmd/prod_test.go
new file mode 100644
index 00000000000..4ce6112122a
--- /dev/null
+++ b/client/go/cmd/prod_test.go
@@ -0,0 +1,166 @@
+package cmd
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/vespa-engine/vespa/client/go/util"
+)
+
+func TestProdInit(t *testing.T) {
+	homeDir := filepath.Join(t.TempDir(), ".vespa")
+	pkgDir := filepath.Join(t.TempDir(), "app")
+	createApplication(t, pkgDir)
+
+	answers := []string{
+		// Regions
+		"invalid input",
+		"aws-us-west-2a,aws-eu-west-1a",
+
+		// Node count: qrs
+		"invalid input",
+		"4",
+
+		// Node resources: qrs
+		"invalid input",
+		"auto",
+
+		// Node count: music
+		"invalid input",
+		"6",
+
+		// Node resources: music
+		"invalid input",
+		"vcpu=16,memory=64Gb,disk=100Gb",
+	}
+	var buf bytes.Buffer
+	buf.WriteString(strings.Join(answers, "\n") + "\n")
+	execute(command{stdin: &buf, homeDir: homeDir, args: []string{"prod", "init", pkgDir}}, t, nil)
+
+	// Verify contents
+	deploymentPath := filepath.Join(pkgDir, "src", "main", "application", "deployment.xml")
+	deploymentXML := readFileString(t, deploymentPath)
+	assert.Contains(t, deploymentXML, `<region active="true">aws-us-west-2a</region>`)
+	assert.Contains(t, deploymentXML, `<region active="true">aws-eu-west-1a</region>`)
+
+	servicesPath := filepath.Join(pkgDir, "src", "main", "application", "services.xml")
+	servicesXML := readFileString(t, servicesPath)
+	containerFragment := `<container id="qrs" version="1.0">
+    <document-api></document-api>
+    <search></search>
+    <nodes count="4"></nodes>
+  </container>`
+	assert.Contains(t, servicesXML, containerFragment)
+	contentFragment := `<content id="music" version="1.0">
+    <redundancy>2</redundancy>
+    <documents>
+      <document type="music" mode="index"></document>
+    </documents>
+    <nodes count="6">
+      <resources vcpu="16" memory="64Gb" disk="100Gb"></resources>
+    </nodes>
+  </content>`
+	assert.Contains(t, servicesXML, contentFragment)
+
+	// Backups are created
+	assert.True(t, util.PathExists(deploymentPath+".1.bak"))
+	assert.True(t, util.PathExists(servicesPath+".1.bak"))
+}
+
+func readFileString(t *testing.T, filename string) string {
+	content, err := ioutil.ReadFile(filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(content)
+}
+
+func createApplication(t *testing.T, pkgDir string) {
+	appDir := filepath.Join(pkgDir, "src", "main", "application")
+	targetDir := filepath.Join(pkgDir, "target")
+	if err := os.MkdirAll(appDir, 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	deploymentXML := `<deployment version="1.0">
+  <prod>
+    <region active="true">aws-us-east-1c</region>
+  </prod>
+</deployment>`
+	if err := ioutil.WriteFile(filepath.Join(appDir, "deployment.xml"), []byte(deploymentXML), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	servicesXML := `<services version="1.0" xmlns:deploy="vespa" xmlns:preprocess="properties">
+  <container id="qrs" version="1.0">
+    <document-api/>
+    <search/>
+    <nodes count="2">
+      <resources vcpu="4" memory="8Gb" disk="100Gb"/>
+    </nodes>
+  </container>
+  <content id="music" version="1.0">
+    <redundancy>2</redundancy>
+    <documents>
+      <document type="music" mode="index"></document>
+    </documents>
+    <nodes count="4"></nodes>
+  </content>
+</services>`
+
+	if err := ioutil.WriteFile(filepath.Join(appDir, "services.xml"), []byte(servicesXML), 0644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.MkdirAll(targetDir, 0755); err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(filepath.Join(pkgDir, "pom.xml"), []byte(""), 0644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestProdSubmit(t *testing.T) {
+	homeDir := filepath.Join(t.TempDir(), ".vespa")
+	pkgDir := filepath.Join(t.TempDir(), "app")
+	createApplication(t, pkgDir)
+
+	httpClient := &mockHttpClient{}
+	httpClient.NextResponse(200, `ok`)
+	execute(command{homeDir: homeDir, args: []string{"config", "set", "application", "t1.a1.i1"}}, t, httpClient)
+	execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "cloud"}}, t, httpClient)
+	execute(command{homeDir: homeDir, args: []string{"api-key"}}, t, httpClient)
+	execute(command{homeDir: homeDir, args: []string{"cert", pkgDir}}, t, httpClient)
+
+	// Copy an application package pre-assambled with mvn package
+	testAppDir := filepath.Join("testdata", "applications", "withDeployment", "target")
+	zipFile := filepath.Join(testAppDir, "application.zip")
+	copyFile(t, filepath.Join(pkgDir, "target", "application.zip"), zipFile)
+	testZipFile := filepath.Join(testAppDir, "application-test.zip")
+	copyFile(t, filepath.Join(pkgDir, "target", "application-test.zip"), testZipFile)
+
+	out, _ := execute(command{homeDir: homeDir, args: []string{"prod", "submit", pkgDir}}, t, httpClient)
+	assert.Contains(t, out, "Success: Submitted")
+	assert.Contains(t, out, "See https://console.vespa.oath.cloud/tenant/t1/application/a1/prod/deployment for deployment progress")
+}
+
+func copyFile(t *testing.T, dstFilename, srcFilename string) {
+	dst, err := os.Create(dstFilename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer dst.Close()
+	src, err := os.Open(srcFilename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer src.Close()
+	if _, err := io.Copy(dst, src); err != nil {
+		t.Fatal(err)
+	}
+}
diff --git a/client/go/cmd/query_test.go b/client/go/cmd/query_test.go
index 81dc03766be..ec6c3063906 100644
--- a/client/go/cmd/query_test.go
+++ b/client/go/cmd/query_test.go
@@ -45,12 +45,12 @@ func TestServerError(t *testing.T) {
 
 func assertQuery(t *testing.T, expectedQuery string, query ...string) {
 	client := &mockHttpClient{}
-	queryURL := queryServiceURL(client)
 	client.NextResponse(200, "{\"query\":\"result\"}")
 	assert.Equal(t,
 		"{\n    \"query\": \"result\"\n}\n",
 		executeCommand(t, client, []string{"query"}, query),
 		"query output")
+	queryURL := queryServiceURL(client)
 	assert.Equal(t, queryURL+"/search/"+expectedQuery, client.lastRequest.URL.String())
 }
 
diff --git a/client/go/cmd/root.go b/client/go/cmd/root.go
index ab6c19a665b..5aae55ab6e4 100644
--- a/client/go/cmd/root.go
+++ b/client/go/cmd/root.go
@@ -6,6 +6,7 @@ package cmd
 
 import (
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"os"
@@ -37,6 +38,7 @@ Vespa documentation: https://docs.vespa.ai`,
 	waitSecsArg    int
 	colorArg       string
 	quietArg       bool
+	stdin          io.ReadWriter = os.Stdin
 
 	color  = aurora.NewAurora(false)
 	stdout = colorable.NewColorableStdout()
diff --git a/client/go/cmd/testdata/applications/withDeployment/target/application-test.zip b/client/go/cmd/testdata/applications/withDeployment/target/application-test.zip
new file mode 100644
index 00000000000..8a1707b9cee
--- /dev/null
+++ b/client/go/cmd/testdata/applications/withDeployment/target/application-test.zip
diff --git a/client/go/cmd/testdata/applications/withDeployment/target/application.zip b/client/go/cmd/testdata/applications/withDeployment/target/application.zip
new file mode 100644
index 00000000000..da23c2ff437
--- /dev/null
+++ b/client/go/cmd/testdata/applications/withDeployment/target/application.zip
diff --git a/client/go/util/io.go b/client/go/util/io.go
index f51c6060cb7..23bfec84879 100644
--- a/client/go/util/io.go
+++ b/client/go/util/io.go
@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"errors"
 	"io"
+	"io/ioutil"
 	"os"
 	"strings"
 )
@@ -41,7 +42,7 @@ func ReaderToBytes(reader io.Reader) []byte {
 
 // Returns the contents of reader as indented JSON
 func ReaderToJSON(reader io.Reader) string {
-	bodyBytes := ReaderToBytes(reader)
+	bodyBytes, _ := ioutil.ReadAll(reader)
 	var prettyJSON bytes.Buffer
 	parseError := json.Indent(&prettyJSON, bodyBytes, "", "    ")
 	if parseError != nil { // Not JSON: Print plainly
@@ -49,3 +50,19 @@ func ReaderToJSON(reader io.Reader) string {
 	}
 	return prettyJSON.String()
 }
+
+// AtomicWriteFile atomically writes data to filename.
+func AtomicWriteFile(filename string, data []byte) error {
+	tmpFile, err := ioutil.TempFile("", "vespa")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(tmpFile.Name())
+	if _, err := tmpFile.Write(data); err != nil {
+		return err
+	}
+	if err := tmpFile.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmpFile.Name(), filename)
+}
diff --git a/client/go/vespa/crypto.go b/client/go/vespa/crypto.go
index b4a5a5b7da8..25d3a937f4b 100644
--- a/client/go/vespa/crypto.go
+++ b/client/go/vespa/crypto.go
@@ -13,21 +13,20 @@ import (
 	"encoding/base64"
 	"encoding/hex"
 	"encoding/pem"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"math/big"
 	"net/http"
-	"os"
 	"strings"
 	"time"
+
+	"github.com/vespa-engine/vespa/client/go/util"
 )
 
 const (
 	defaultCommonName = "cloud.vespa.example"
 	certificateExpiry = 3650 * 24 * time.Hour // Approximately 10 years
-	tempFilePattern   = "vespa"
 )
 
 // PemKeyPair represents a PEM-encoded private key and X509 certificate.
@@ -38,31 +37,18 @@ type PemKeyPair struct {
 
 // WriteCertificateFile writes the certificate contained in this key pair to certificateFile.
 func (kp *PemKeyPair) WriteCertificateFile(certificateFile string, overwrite bool) error {
-	return atomicWriteFile(certificateFile, kp.Certificate, overwrite)
+	if util.PathExists(certificateFile) && !overwrite {
+		return fmt.Errorf("cannot overwrite existing file: %s", certificateFile)
+	}
+	return util.AtomicWriteFile(certificateFile, kp.Certificate)
 }
 
 // WritePrivateKeyFile writes the private key contained in this key pair to privateKeyFile.
 func (kp *PemKeyPair) WritePrivateKeyFile(privateKeyFile string, overwrite bool) error {
-	return atomicWriteFile(privateKeyFile, kp.PrivateKey, overwrite)
-}
-
-func atomicWriteFile(filename string, data []byte, overwrite bool) error {
-	tmpFile, err := ioutil.TempFile("", tempFilePattern)
-	if err != nil {
-		return err
-	}
-	defer os.Remove(tmpFile.Name())
-	if _, err := tmpFile.Write(data); err != nil {
-		return err
-	}
-	if err := tmpFile.Close(); err != nil {
-		return err
-	}
-	_, err = os.Stat(filename)
-	if errors.Is(err, os.ErrNotExist) || overwrite {
-		return os.Rename(tmpFile.Name(), filename)
+	if util.PathExists(privateKeyFile) && !overwrite {
+		return fmt.Errorf("cannot overwrite existing file: %s", privateKeyFile)
 	}
-	return fmt.Errorf("cannot overwrite existing file: %s", filename)
+	return util.AtomicWriteFile(privateKeyFile, kp.PrivateKey)
 }
 
 // CreateKeyPair creates a key pair containing a private key and self-signed X509 certificate.
diff --git a/client/go/vespa/deploy.go b/client/go/vespa/deploy.go
index eec0182b0ce..3718c7d813a 100644
--- a/client/go/vespa/deploy.go
+++ b/client/go/vespa/deploy.go
@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"mime/multipart"
 	"net/http"
 	"net/url"
 	"os"
@@ -49,7 +50,8 @@ type DeploymentOpts struct {
 }
 
 type ApplicationPackage struct {
-	Path string
+	Path     string
+	TestPath string
 }
 
 func (a ApplicationID) String() string {
@@ -79,6 +81,15 @@ func (d *DeploymentOpts) url(path string) (*url.URL, error) {
 }
 
 func (ap *ApplicationPackage) HasCertificate() bool {
+	return ap.hasFile(filepath.Join("security", "clients.pem"), "security/clients.pem")
+}
+
+func (ap *ApplicationPackage) HasDeployment() bool { return ap.hasFile("deployment.xml", "") }
+
+func (ap *ApplicationPackage) hasFile(filename, zipName string) bool {
+	if zipName == "" {
+		zipName = filename
+	}
 	if ap.IsZip() {
 		r, err := zip.OpenReader(ap.Path)
 		if err != nil {
@@ -86,35 +97,58 @@ func (ap *ApplicationPackage) HasCertificate() bool {
 		}
 		defer r.Close()
 		for _, f := range r.File {
-			if f.Name == "security/clients.pem" {
+			if f.Name == zipName {
 				return true
 			}
 		}
 		return false
 	}
-	return util.PathExists(filepath.Join(ap.Path, "security", "clients.pem"))
+	return util.PathExists(filepath.Join(ap.Path, filename))
 }
 
 func (ap *ApplicationPackage) IsZip() bool { return isZip(ap.Path) }
 
-func (ap *ApplicationPackage) zipReader() (io.ReadCloser, error) {
+func (ap *ApplicationPackage) IsJava() bool {
+	if ap.IsZip() {
+		r, err := zip.OpenReader(ap.Path)
+		if err != nil {
+			return false
+		}
+		defer r.Close()
+		for _, f := range r.File {
+			if filepath.Ext(f.Name) == ".jar" {
+				return true
+			}
+		}
+		return false
+	}
+	return util.PathExists(filepath.Join(ap.Path, "pom.xml"))
+}
+
+func (ap *ApplicationPackage) zipReader(test bool) (io.ReadCloser, error) {
 	zipFile := ap.Path
+	if test {
+		zipFile = ap.TestPath
+	}
 	if !ap.IsZip() {
-		tempZip, error := ioutil.TempFile("", "application.zip")
-		if error != nil {
-			return nil, fmt.Errorf("Could not create a temporary zip file for the application package: %w", error)
+		tempZip, err := ioutil.TempFile("", "vespa")
+		if err != nil {
+			return nil, fmt.Errorf("Could not create a temporary zip file for the application package: %w", err)
 		}
+		defer func() {
+			tempZip.Close()
+			os.Remove(tempZip.Name())
+		}()
 		if err := zipDir(ap.Path, tempZip.Name()); err != nil {
 			return nil, err
 		}
-		defer os.Remove(tempZip.Name())
 		zipFile = tempZip.Name()
 	}
-	r, err := os.Open(zipFile)
+	f, err := os.Open(zipFile)
 	if err != nil {
 		return nil, fmt.Errorf("Could not open application package at %s: %w", ap.Path, err)
 	}
-	return r, nil
+	return f, nil
 }
 
 // FindApplicationPackage finds the path to an application package from the zip file or directory zipOrDir.
@@ -125,7 +159,8 @@ func FindApplicationPackage(zipOrDir string, requirePackaging bool) (Application
 	if util.PathExists(filepath.Join(zipOrDir, "pom.xml")) {
 		zip := filepath.Join(zipOrDir, "target", "application.zip")
 		if util.PathExists(zip) {
-			return ApplicationPackage{Path: zip}, nil
+			testZip := filepath.Join(zipOrDir, "target", "application-test.zip")
+			return ApplicationPackage{Path: zip, TestPath: testZip}, nil
 		}
 		if requirePackaging {
 			return ApplicationPackage{}, errors.New("pom.xml exists but no target/application.zip. Run mvn package first")
@@ -214,11 +249,8 @@ func Activate(sessionID int64, deployment DeploymentOpts) error {
 func Deploy(opts DeploymentOpts) (int64, error) {
 	path := "/application/v2/tenant/default/prepareandactivate"
 	if opts.IsCloud() {
-		if !opts.ApplicationPackage.HasCertificate() {
-			return 0, fmt.Errorf("%s: missing certificate in package", opts)
-		}
-		if opts.APIKey == nil {
-			return 0, fmt.Errorf("%s: missing api key", opts.String())
+		if err := checkDeploymentOpts(opts); err != nil {
+			return 0, err
 		}
 		if opts.Deployment.Zone.Environment == "" || opts.Deployment.Zone.Region == "" {
 			return 0, fmt.Errorf("%s: missing zone", opts)
@@ -237,8 +269,89 @@ func Deploy(opts DeploymentOpts) (int64, error) {
 	return uploadApplicationPackage(u, opts)
 }
 
+func copyToPart(dst *multipart.Writer, src io.Reader, fieldname, filename string) error {
+	var part io.Writer
+	var err error
+	if filename == "" {
+		part, err = dst.CreateFormField(fieldname)
+	} else {
+		part, err = dst.CreateFormFile(fieldname, filename)
+	}
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(part, src); err != nil {
+		return err
+	}
+	return nil
+}
+
+func Submit(opts DeploymentOpts) error {
+	if !opts.IsCloud() {
+		return fmt.Errorf("%s: submit is unsupported", opts)
+	}
+	if err := checkDeploymentOpts(opts); err != nil {
+		return err
+	}
+	path := fmt.Sprintf("/application/v4/tenant/%s/application/%s/submit", opts.Deployment.Application.Tenant, opts.Deployment.Application.Application)
+	u, err := opts.url(path)
+	if err != nil {
+		return err
+	}
+	var body bytes.Buffer
+	writer := multipart.NewWriter(&body)
+	if err := copyToPart(writer, strings.NewReader("{}"), "submitOptions", ""); err != nil {
+		return err
+	}
+	applicationZip, err := opts.ApplicationPackage.zipReader(false)
+	if err != nil {
+		return err
+	}
+	if err := copyToPart(writer, applicationZip, "applicationZip", "application.zip"); err != nil {
+		return err
+	}
+	testApplicationZip, err := opts.ApplicationPackage.zipReader(true)
+	if err != nil {
+		return err
+	}
+	if err := copyToPart(writer, testApplicationZip, "applicationTestZip", "application-test.zip"); err != nil {
+		return err
+	}
+	if err := writer.Close(); err != nil {
+		return err
+	}
+	request := &http.Request{
+		URL:    u,
+		Method: "POST",
+		Body:   ioutil.NopCloser(&body),
+		Header: make(http.Header),
+	}
+	request.Header.Set("Content-Type", writer.FormDataContentType())
+	signer := NewRequestSigner(opts.Deployment.Application.SerializedForm(), opts.APIKey)
+	if err := signer.SignRequest(request); err != nil {
+		return err
+	}
+	serviceDescription := "Submit service"
+	response, err := util.HttpDo(request, time.Minute*10, serviceDescription)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+	return checkResponse(request, response, serviceDescription)
+}
+
+func checkDeploymentOpts(opts DeploymentOpts) error {
+	if !opts.ApplicationPackage.HasCertificate() {
+		return fmt.Errorf("%s: missing certificate in package", opts)
+	}
+	if opts.APIKey == nil {
+		return fmt.Errorf("%s: missing api key", opts.String())
+	}
+	return nil
+}
+
 func uploadApplicationPackage(url *url.URL, opts DeploymentOpts) (int64, error) {
-	zipReader, err := opts.ApplicationPackage.zipReader()
+	zipReader, err := opts.ApplicationPackage.zipReader(false)
 	if err != nil {
 		return 0, err
 	}
@@ -344,7 +457,7 @@ func zipDir(dir string, destination string) error {
 
 // Returns the error message in the given JSON, or the entire content if it could not be extracted
 func extractError(reader io.Reader) string {
-	responseData := util.ReaderToBytes(reader)
+	responseData, _ := ioutil.ReadAll(reader)
 	var response map[string]interface{}
 	json.Unmarshal(responseData, &response)
 	if response["error-code"] == "INVALID_APPLICATION_PACKAGE" {
diff --git a/client/go/vespa/document.go b/client/go/vespa/document.go
index 5e01d180b5f..6424113bd52 100644
--- a/client/go/vespa/document.go
+++ b/client/go/vespa/document.go
@@ -59,12 +59,15 @@ func sendOperation(documentId string, jsonFile string, service *Service, operati
 	if operation == "remove" && jsonFile == "" {
 		documentData = []byte("{\n    \"remove\": \"" + documentId + "\"\n}\n")
 	} else {
-		fileReader, fileError := os.Open(jsonFile)
-		if fileError != nil {
-			return util.FailureWithDetail("Could not open file '"+jsonFile+"'", fileError.Error())
+		fileReader, err := os.Open(jsonFile)
+		if err != nil {
+			return util.FailureWithDetail("Could not open file '"+jsonFile+"'", err.Error())
 		}
 		defer fileReader.Close()
-		documentData = util.ReaderToBytes(fileReader)
+		documentData, err = ioutil.ReadAll(fileReader)
+		if err != nil {
+			return util.FailureWithDetail("Failed to read '"+jsonFile+"'", err.Error())
+		}
 	}
 
 	var doc map[string]interface{}
diff --git a/client/go/vespa/xml/config.go b/client/go/vespa/xml/config.go
new file mode 100644
index 00000000000..e900b50cbb0
--- /dev/null
+++ b/client/go/vespa/xml/config.go
@@ -0,0 +1,341 @@
+package xml
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+var DefaultDeployment Deployment
+
+func init() {
+	defaultDeploymentRaw := `<deployment version="1.0">
+  <prod>
+    <region active="true">aws-us-east-1c</region>
+  </prod>
+</deployment>`
+	d, err := ReadDeployment(strings.NewReader(defaultDeploymentRaw))
+	if err != nil {
+		panic(err)
+	}
+	DefaultDeployment = d
+}
+
+// Deployment represents the contents of a deployment.xml file.
+type Deployment struct {
+	Root     xml.Name   `xml:"deployment"`
+	Version  string     `xml:"version,attr"`
+	Instance []Instance `xml:"instance"`
+	Prod     Prod       `xml:"prod"`
+	rawXML   bytes.Buffer
+}
+
+type Instance struct {
+	Prod Prod `xml:"prod"`
+}
+
+type Prod struct {
+	Regions []Region `xml:"region"`
+}
+
+type Region struct {
+	Name   string `xml:",chardata"`
+	Active bool   `xml:"active,attr"`
+}
+
+func (d Deployment) String() string { return d.rawXML.String() }
+
+// Replace replaces any elements of name found under parentName with data.
+func (s *Deployment) Replace(parentName, name string, data interface{}) error {
+	rewritten, err := Replace(&s.rawXML, parentName, name, data)
+	if err != nil {
+		return err
+	}
+	newXML, err := ReadDeployment(strings.NewReader(rewritten))
+	if err != nil {
+		return err
+	}
+	*s = newXML
+	return nil
+}
+
+// Services represents the contents of a services.xml file.
+type Services struct {
+	Root      xml.Name    `xml:"services"`
+	Container []Container `xml:"container"`
+	Content   []Content   `xml:"content"`
+	rawXML    bytes.Buffer
+}
+
+type Container struct {
+	Root  xml.Name `xml:"container"`
+	ID    string   `xml:"id,attr"`
+	Nodes Nodes    `xml:"nodes"`
+}
+
+type Content struct {
+	ID    string `xml:"id,attr"`
+	Nodes Nodes  `xml:"nodes"`
+}
+
+type Nodes struct {
+	Count     string     `xml:"count,attr"`
+	Resources *Resources `xml:"resources,omitempty"`
+}
+
+type Resources struct {
+	Vcpu   string `xml:"vcpu,attr"`
+	Memory string `xml:"memory,attr"`
+	Disk   string `xml:"disk,attr"`
+}
+
+func (s Services) String() string { return s.rawXML.String() }
+
+// Replace replaces any elements of name found under parentName with data.
+func (s *Services) Replace(parentName, name string, data interface{}) error {
+	rewritten, err := Replace(&s.rawXML, parentName, name, data)
+	if err != nil {
+		return err
+	}
+	newXML, err := ReadServices(strings.NewReader(rewritten))
+	if err != nil {
+		return err
+	}
+	*s = newXML
+	return nil
+}
+
+func (r Resources) String() string {
+	return fmt.Sprintf("vcpu=%s,memory=%s,disk=%s", r.Vcpu, r.Memory, r.Disk)
+}
+
+// ReadDeployment reads deployment.xml from reader r.
+func ReadDeployment(r io.Reader) (Deployment, error) {
+	var deployment Deployment
+	var rawXML bytes.Buffer
+	dec := xml.NewDecoder(io.TeeReader(r, &rawXML))
+	if err := dec.Decode(&deployment); err != nil {
+		return Deployment{}, err
+	}
+	deployment.rawXML = rawXML
+	return deployment, nil
+}
+
+// ReadServices reads services.xml from reader r.
+func ReadServices(r io.Reader) (Services, error) {
+	var services Services
+	var rawXML bytes.Buffer
+	dec := xml.NewDecoder(io.TeeReader(r, &rawXML))
+	if err := dec.Decode(&services); err != nil {
+		return Services{}, err
+	}
+	services.rawXML = rawXML
+	return services, nil
+}
+
+// Regions returns given region names as elements.
+func Regions(names ...string) []Region {
+	var regions []Region
+	for _, z := range names {
+		regions = append(regions, Region{Name: z, Active: true})
+	}
+	return regions
+}
+
+// ParseResources parses nodes resources from string s.
+func ParseResources(s string) (Resources, error) {
+	parts := strings.Split(s, ",")
+	if len(parts) != 3 {
+		return Resources{}, fmt.Errorf("invalid resources: %q", s)
+	}
+	vcpu, err := parseResource("vcpu", parts[0])
+	if err != nil {
+		return Resources{}, err
+	}
+	memory, err := parseResource("memory", parts[1])
+	if err != nil {
+		return Resources{}, err
+	}
+	disk, err := parseResource("disk", parts[2])
+	if err != nil {
+		return Resources{}, err
+	}
+	return Resources{Vcpu: vcpu, Memory: memory, Disk: disk}, nil
+}
+
+// ParseNodeCount parses a node count range from string s.
+func ParseNodeCount(s string) (int, int, error) {
+	parseErr := fmt.Errorf("invalid node count: %q", s)
+	n, err := strconv.Atoi(s)
+	if err == nil {
+		return n, n, nil
+	}
+	if strings.HasPrefix(s, "[") && strings.HasSuffix(s, "]") {
+		parts := strings.Split(s[1:len(s)-1], ",")
+		if len(parts) != 2 {
+			return 0, 0, parseErr
+		}
+		min, err := strconv.Atoi(parts[0])
+		if err != nil {
+			return 0, 0, parseErr
+		}
+		max, err := strconv.Atoi(parts[1])
+		if err != nil {
+			return 0, 0, parseErr
+		}
+		return min, max, nil
+	}
+	return 0, 0, parseErr
+}
+
+// IsProdRegion returns whether string s is a valid production region.
+func IsProdRegion(s string, system string) bool {
+	if system == "publiccd" {
+		return s == "aws-us-east-1c"
+	}
+	switch s {
+	case "aws-us-east-1c", "aws-us-west-2a",
+		"aws-eu-west-1a", "aws-ap-northeast-1a":
+		return true
+	}
+	return false
+}
+
+func parseResource(field, s string) (string, error) {
+	parts := strings.SplitN(s, "=", 2)
+	if len(parts) != 2 || parts[0] != field {
+		return "", fmt.Errorf("invalid value for %s field: %q", field, s)
+	}
+	return parts[1], nil
+}
+
+// ReplaceRaw finds all elements of name in rawXML and replaces their contents with value.
+func ReplaceRaw(rawXML, name, value string) string {
+	startElement := "<" + name + ">"
+	endElement := "</" + name + ">"
+	re := regexp.MustCompile(regexp.QuoteMeta(startElement) + ".*" + regexp.QuoteMeta(endElement))
+	return re.ReplaceAllString(rawXML, startElement+value+endElement)
+}
+
+// Replace looks for an element name in the XML read from reader r, appearing inside a element named parentName.
+//
+// Any matching elements found are replaced with data. If parentName contains an ID selector, e.g. "email#my-id", only
+// the elements inside the parent element with the attribute id="my-id" are replaced.
+//
+// If data is nil, any matching elements are removed instead of replaced.
+func Replace(r io.Reader, parentName, name string, data interface{}) (string, error) {
+	var buf bytes.Buffer
+	dec := xml.NewDecoder(r)
+	enc := xml.NewEncoder(&buf)
+	enc.Indent("", "  ")
+
+	parts := strings.SplitN(parentName, "#", 2)
+	id := ""
+	if len(parts) > 1 {
+		parentName = parts[0]
+		id = parts[1]
+	}
+
+	foundParent := false
+	replacing := false
+	done := false
+	for {
+		token, err := dec.Token()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return "", err
+		}
+		token = joinNamespace(token)
+		if isEndElement(parentName, token) {
+			foundParent = false
+			done = false
+		}
+		if _, ok := getStartElement(parentName, id, token); ok {
+			foundParent = true
+		}
+		if foundParent {
+			if isEndElement(name, token) {
+				replacing = false
+				continue
+			}
+			replacableElement, ok := getStartElement(name, "", token)
+			if ok {
+				replacing = true
+			}
+			if replacing {
+				if !done && data != nil {
+					replacableElement.Attr = nil // Clear any existing attributes as given data should contain the wanted ones
+					if err := enc.EncodeElement(data, replacableElement); err != nil {
+						return "", err
+					}
+					done = true
+				}
+				continue
+			}
+		}
+		if err := enc.EncodeToken(token); err != nil {
+			return "", err
+		}
+	}
+	if err := enc.Flush(); err != nil {
+		return "", err
+	}
+	var sb strings.Builder
+	scanner := bufio.NewScanner(&buf)
+	for scanner.Scan() {
+		line := scanner.Text()
+		// Skip lines containing only whitespace
+		if strings.TrimSpace(line) == "" {
+			continue
+		}
+		sb.WriteString(line)
+		sb.WriteRune('\n')
+	}
+	if err := scanner.Err(); err != nil {
+		return "", err
+	}
+	return sb.String(), nil
+}
+
+func joinNamespace(token xml.Token) xml.Token {
+	// Hack to work around the broken namespace support in Go
+	// https://github.com/golang/go/issues/13400
+	if startElement, ok := token.(xml.StartElement); ok {
+		attr := make([]xml.Attr, 0, len(startElement.Attr))
+		for _, a := range startElement.Attr {
+			if a.Name.Space != "" {
+				a.Name.Space = ""
+				a.Name.Local = "xmlns:" + a.Name.Local
+			}
+			attr = append(attr, a)
+		}
+		startElement.Attr = attr
+		return startElement
+	}
+	return token
+}
+
+func getStartElement(name, id string, token xml.Token) (xml.StartElement, bool) {
+	startElement, ok := token.(xml.StartElement)
+	if !ok {
+		return xml.StartElement{}, false
+	}
+	matchingID := id == ""
+	for _, attr := range startElement.Attr {
+		if attr.Name.Local == "id" && attr.Value == id {
+			matchingID = true
+		}
+	}
+	return startElement, startElement.Name.Local == name && matchingID
+}
+
+func isEndElement(name, token xml.Token) bool {
+	endElement, ok := token.(xml.EndElement)
+	return ok && endElement.Name.Local == name
+}
diff --git a/client/go/vespa/xml/config_test.go b/client/go/vespa/xml/config_test.go
new file mode 100644
index 00000000000..9d18636473b
--- /dev/null
+++ b/client/go/vespa/xml/config_test.go
@@ -0,0 +1,297 @@
+package xml
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestReplaceDeployment(t *testing.T) {
+	in := `
+<deployment version="1.0">
+    <prod>
+        <region active="true">us-north-1</region>
+        <region active="false">eu-north-2</region>
+    </prod>
+</deployment>`
+
+	out := `<deployment version="1.0">
+  <prod>
+    <region active="true">eu-south-1</region>
+    <region active="true">us-central-1</region>
+  </prod>
+</deployment>
+`
+	regions := Regions("eu-south-1", "us-central-1")
+	assertReplace(t, in, out, "prod", "region", regions)
+}
+
+func TestReplaceDeploymentWithInstance(t *testing.T) {
+	in := `
+<deployment version="1.0">
+    <instance id="default">
+        <prod>
+            <region active="true">us-north-1</region>
+        </prod>
+    </instance>
+    <instance id="beta">
+        <prod>
+            <region active="true">eu-south-1</region>
+        </prod>
+    </instance>
+</deployment>`
+
+	out := `<deployment version="1.0">
+  <instance id="default">
+    <prod>
+      <region active="true">us-central-1</region>
+      <region active="true">eu-west-1</region>
+    </prod>
+  </instance>
+  <instance id="beta">
+    <prod>
+      <region active="true">us-central-1</region>
+      <region active="true">eu-west-1</region>
+    </prod>
+  </instance>
+</deployment>
+`
+	regions := Regions("us-central-1", "eu-west-1")
+	assertReplace(t, in, out, "prod", "region", regions)
+}
+
+func TestReplaceServices(t *testing.T) {
+	in := `
+<services xmlns:deploy="vespa" xmlns:preprocess="properties">
+  <container id="qrs">
+    <search/>
+    <document-api/>
+    <nodes count="2">
+      <resources vcpu="4" memory="8Gb" disk="50Gb"/>
+    </nodes>
+  </container>
+  <content id="music">
+    <redundancy>2</redundancy>
+    <nodes count="3">
+      <resources vcpu="8" memory="32Gb" disk="200Gb"/>
+    </nodes>
+    <documents>
+      <document type="music"/>
+    </documents>
+  </content>
+</services>
+`
+
+	out := `<services xmlns:deploy="vespa" xmlns:preprocess="properties">
+  <container id="qrs">
+    <search></search>
+    <document-api></document-api>
+    <nodes count="4">
+      <resources vcpu="2" memory="4Gb" disk="50Gb"></resources>
+    </nodes>
+  </container>
+  <content id="music">
+    <redundancy>2</redundancy>
+    <nodes count="3">
+      <resources vcpu="8" memory="32Gb" disk="200Gb"></resources>
+    </nodes>
+    <documents>
+      <document type="music"></document>
+    </documents>
+  </content>
+</services>
+`
+	nodes := Nodes{Count: "4", Resources: &Resources{Vcpu: "2", Memory: "4Gb", Disk: "50Gb"}}
+	assertReplace(t, in, out, "container#qrs", "nodes", nodes)
+}
+
+func TestReplaceServicesEmptyResources(t *testing.T) {
+	in := `<services xmlns:deploy="vespa" xmlns:preprocess="properties">
+  <container id="movies">
+    <search></search>
+    <document-api></document-api>
+    <nodes count="4"/>
+  </container>
+</services>
+`
+	out := `<services xmlns:deploy="vespa" xmlns:preprocess="properties">
+  <container id="movies">
+    <search></search>
+    <document-api></document-api>
+    <nodes count="5">
+      <resources vcpu="4" memory="8Gb" disk="100Gb"></resources>
+    </nodes>
+  </container>
+</services>
+`
+	nodes := Nodes{Count: "5", Resources: &Resources{Vcpu: "4", Memory: "8Gb", Disk: "100Gb"}}
+	assertReplace(t, in, out, "container#movies", "nodes", nodes)
+}
+
+func TestReplaceRemovesElement(t *testing.T) {
+	in := `
+<deployment version="1.0">
+    <prod>
+        <region active="true">eu-south-1</region>
+        <region active="true">us-central-1</region>
+        <test>us-central-1</test>
+    </prod>
+</deployment>`
+
+	out := `<deployment version="1.0">
+  <prod>
+    <region active="true">eu-south-1</region>
+    <region active="true">us-central-1</region>
+  </prod>
+</deployment>
+`
+	assertReplace(t, in, out, "prod", "test", nil)
+}
+
+func TestReplaceRaw(t *testing.T) {
+	in := `
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+                             http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>ai.vespa.cloud.docsearch</groupId>
+    <artifactId>vespacloud-docsearch</artifactId>
+    <packaging>container-plugin</packaging>
+    <version>1.0.0</version>
+
+    <parent>
+        <groupId>com.yahoo.vespa</groupId>
+        <artifactId>cloud-tenant-base</artifactId>
+        <version>[7,999)</version>
+    </parent>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <tenant>tenant</tenant>
+        <application>app</application>
+        <instance>instance</instance>
+    </properties>
+
+</project>
+`
+	replacements := map[string]string{
+		"tenant":      "vespa-team",
+		"application": "music",
+		"instance":    "default",
+	}
+	rewritten := in
+	for element, value := range replacements {
+		rewritten = ReplaceRaw(rewritten, element, value)
+	}
+
+	out := `
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+                             http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>ai.vespa.cloud.docsearch</groupId>
+    <artifactId>vespacloud-docsearch</artifactId>
+    <packaging>container-plugin</packaging>
+    <version>1.0.0</version>
+
+    <parent>
+        <groupId>com.yahoo.vespa</groupId>
+        <artifactId>cloud-tenant-base</artifactId>
+        <version>[7,999)</version>
+    </parent>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <tenant>vespa-team</tenant>
+        <application>music</application>
+        <instance>default</instance>
+    </properties>
+
+</project>
+`
+	if rewritten != out {
+		t.Errorf("got:\n%s\nwant:\n%s\n", rewritten, out)
+	}
+}
+
+func TestReadServicesNoResources(t *testing.T) {
+	s := `
+<services xmlns:deploy="vespa" xmlns:preprocess="properties">
+  <container id="qrs">
+     <nodes count="2">
+      <resources vcpu="4" memory="8Gb" disk="50Gb"/>
+    </nodes>
+  </container>
+  <content id="music">
+    <redundancy>2</redundancy>
+    <nodes count="3"/>
+    <documents>
+      <document type="music"/>
+    </documents>
+  </content>
+</services>
+`
+	services, err := ReadServices(strings.NewReader(s))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := services.Content[0].Nodes.Resources; got != nil {
+		t.Errorf("got %+v, want nil", got)
+	}
+}
+
+func TestParseResources(t *testing.T) {
+	assertResources(t, "foo", Resources{}, true)
+	assertResources(t, "vcpu=2,memory=4Gb", Resources{}, true)
+	assertResources(t, "memory=4Gb,vcpu=2,disk=100Gb", Resources{}, true)
+	assertResources(t, "vcpu=2,memory=4Gb,disk=100Gb", Resources{Vcpu: "2", Memory: "4Gb", Disk: "100Gb"}, false)
+}
+
+func TestParseNodeCount(t *testing.T) {
+	assertNodeCount(t, "2", 2, 2, false)
+	assertNodeCount(t, "[4,8]", 4, 8, false)
+
+	assertNodeCount(t, "foo", 0, 0, true)
+	assertNodeCount(t, "[foo,bar]", 0, 0, true)
+}
+
+func assertReplace(t *testing.T, input, want, parentElement, element string, data interface{}) {
+	got, err := Replace(strings.NewReader(input), parentElement, element, data)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got != want {
+		t.Errorf("got:\n%s\nwant:\n%s\n", got, want)
+	}
+}
+
+func assertNodeCount(t *testing.T, input string, wantMin, wantMax int, wantErr bool) {
+	min, max, err := ParseNodeCount(input)
+	if wantErr {
+		if err == nil {
+			t.Errorf("want error for input %q", input)
+		}
+		return
+	}
+	if min != wantMin || max != wantMax {
+		t.Errorf("got min = %d, max = %d, want min = %d, max = %d", min, max, wantMin, wantMax)
+	}
+}
+
+func assertResources(t *testing.T, input string, want Resources, wantErr bool) {
+	got, err := ParseResources(input)
+	if wantErr {
+		if err == nil {
+			t.Errorf("want error for %q", input)
+		}
+		return
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(want, got) {
+		t.Errorf("got %+v, want %+v", got, want)
+	}
+}