Implement vespa prod init

5 files changed, 408 insertions, 1 deletions

diff --git a/client/go/cmd/command_tester.go b/client/go/cmd/command_tester.go
index 8eaf6be2c22..201adb8cab7 100644
--- a/client/go/cmd/command_tester.go
+++ b/client/go/cmd/command_tester.go
@@ -7,6 +7,7 @@ package cmd
 import (
 	"bytes"
 	"crypto/tls"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -23,6 +24,7 @@ import (
 type command struct {
 	homeDir  string
 	cacheDir string
+	stdin    io.ReadWriter
 	args     []string
 	moreArgs []string
 }
@@ -67,6 +69,11 @@ func execute(cmd command, t *testing.T, client *mockHttpClient) (string, string)
 	var capturedErr bytes.Buffer
 	stdout = &capturedOut
 	stderr = &capturedErr
+	if cmd.stdin != nil {
+		stdin = cmd.stdin
+	} else {
+		stdin = os.Stdin
+	}
 
 	// Execute command and return output
 	rootCmd.SetArgs(append(cmd.args, cmd.moreArgs...))
diff --git a/client/go/cmd/prod.go b/client/go/cmd/prod.go
new file mode 100644
index 00000000000..15d9edc18a7
--- /dev/null
+++ b/client/go/cmd/prod.go
@@ -0,0 +1,277 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package cmd
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"github.com/vespa-engine/vespa/client/go/util"
+	"github.com/vespa-engine/vespa/client/go/vespa"
+	"github.com/vespa-engine/vespa/client/go/vespa/xml"
+)
+
+func init() {
+	rootCmd.AddCommand(prodCmd)
+	prodCmd.AddCommand(prodInitCmd)
+}
+
+var prodCmd = &cobra.Command{
+	Use:   "prod",
+	Short: "Deploy an application package to production in Vespa Cloud",
+	Long: `Deploy an application package to production in Vespa Cloud.
+
+Configure and deploy your application package to production in Vespa Cloud.`,
+	Example: `$ vespa prod init
+$ vespa prod submit`,
+	DisableAutoGenTag: true,
+	Run: func(cmd *cobra.Command, args []string) {
+		// Root command does nothing
+		cmd.Help()
+		exitFunc(1)
+	},
+}
+
+var prodInitCmd = &cobra.Command{
+	Use:   "init",
+	Short: "Modify service.xml and deployment.xml for production deployment",
+	Long: `Modify service.xml and deployment.xml for production deployment.
+
+Only basic deployment configuration is available through this command. For
+advanced configuration see the relevant Vespa Cloud documentation and make
+changes to deployment.xml and services.xml directly.
+
+Reference:
+https://cloud.vespa.ai/en/reference/services
+https://cloud.vespa.ai/en/reference/deployment`,
+	DisableAutoGenTag: true,
+	Run: func(cmd *cobra.Command, args []string) {
+		appSource := applicationSource(args)
+		pkg, err := vespa.FindApplicationPackage(appSource, false)
+		if err != nil {
+			fatalErr(err)
+			return
+		}
+		if pkg.IsZip() {
+			fatalErrHint(fmt.Errorf("Cannot modify compressed application package %s", pkg.Path),
+				"Try running 'mvn clean' and re-running this command")
+			return
+		}
+
+		deploymentXML, err := readDeploymentXML(pkg)
+		if err != nil {
+			fatalErr(err, "Could not read deployment.xml")
+			return
+		}
+		servicesXML, err := readServicesXML(pkg)
+		if err != nil {
+			fatalErr(err, "A services.xml declaring your cluster(s) must exist")
+			return
+		}
+
+		fmt.Fprint(stdout, "This will modify any existing ", color.Yellow("deployment.xml"), " and ", color.Yellow("services.xml"),
+			"!\nBefore modification a backup of the original file will be created.\n\n")
+		fmt.Fprint(stdout, "A default value is suggested (shown inside brackets) based on\nthe files' existing contents. Press enter to use it.\n\n")
+		fmt.Fprint(stdout, "Abort the configuration at any time by pressing Ctrl-C. The\nfiles will remain untouched.\n\n")
+		r := bufio.NewReader(stdin)
+		deploymentXML = updateRegions(r, deploymentXML)
+		servicesXML = updateNodes(r, servicesXML)
+
+		fmt.Fprintln(stdout)
+		if err := writeWithBackup(pkg, "deployment.xml", deploymentXML.String()); err != nil {
+			fatalErr(err)
+			return
+		}
+		if err := writeWithBackup(pkg, "services.xml", servicesXML.String()); err != nil {
+			fatalErr(err)
+			return
+		}
+	},
+}
+
+func writeWithBackup(pkg vespa.ApplicationPackage, filename, contents string) error {
+	dst := filepath.Join(pkg.Path, filename)
+	if util.PathExists(dst) {
+		data, err := ioutil.ReadFile(dst)
+		if err != nil {
+			return err
+		}
+		if bytes.Equal(data, []byte(contents)) {
+			fmt.Fprintf(stdout, "Not writing %s: File is unchanged\n", color.Yellow(filename))
+			return nil
+		}
+		renamed := false
+		for i := 1; i <= 1000; i++ {
+			bak := fmt.Sprintf("%s.%d.bak", dst, i)
+			if !util.PathExists(bak) {
+				fmt.Fprintf(stdout, "Backing up existing %s to %s\n", color.Yellow(filename), color.Yellow(bak))
+				if err := os.Rename(dst, bak); err != nil {
+					return err
+				}
+				renamed = true
+				break
+			}
+		}
+		if !renamed {
+			return fmt.Errorf("could not find an unused backup name for %s", dst)
+		}
+	}
+	fmt.Fprintf(stdout, "Writing %s\n", color.Green(dst))
+	return ioutil.WriteFile(dst, []byte(contents), 0644)
+}
+
+func updateRegions(r *bufio.Reader, deploymentXML xml.Deployment) xml.Deployment {
+	regions := promptRegions(r, deploymentXML)
+	parts := strings.Split(regions, ",")
+	regionElements := xml.Regions(parts...)
+	if err := deploymentXML.Replace("prod", "region", regionElements); err != nil {
+		fatalErr(err, "Could not update region elements in deployment.xml")
+	}
+	return deploymentXML
+}
+
+func promptRegions(r *bufio.Reader, deploymentXML xml.Deployment) string {
+	fmt.Fprintln(stdout, color.Cyan("> Deployment regions"))
+	fmt.Fprintf(stdout, "Documentation: %s\n", color.Green("https://cloud.vespa.ai/en/reference/zones"))
+	fmt.Fprintf(stdout, "Example: %s\n\n", color.Yellow("aws-us-east-1c,aws-us-west-2a"))
+	var currentRegions []string
+	for _, r := range deploymentXML.Prod.Regions {
+		currentRegions = append(currentRegions, r.Name)
+	}
+	if len(deploymentXML.Instance) > 0 {
+		for _, r := range deploymentXML.Instance[0].Prod.Regions {
+			currentRegions = append(currentRegions, r.Name)
+		}
+	}
+	validator := func(input string) error {
+		regions := strings.Split(input, ",")
+		for _, r := range regions {
+			if !xml.ValidProdRegion(r) {
+				return fmt.Errorf("invalid region %s", r)
+			}
+		}
+		return nil
+	}
+	return prompt(r, "Which regions do you wish to deploy in?", strings.Join(currentRegions, ","), validator)
+}
+
+func updateNodes(r *bufio.Reader, servicesXML xml.Services) xml.Services {
+	for _, c := range servicesXML.Container {
+		nodes := promptNodes(r, c.ID, c.Nodes)
+		if err := servicesXML.Replace("container#"+c.ID, "nodes", nodes); err != nil {
+			fatalErr(err)
+			return xml.Services{}
+		}
+	}
+	for _, c := range servicesXML.Content {
+		nodes := promptNodes(r, c.ID, c.Nodes)
+		if err := servicesXML.Replace("content#"+c.ID, "nodes", nodes); err != nil {
+			fatalErr(err)
+			return xml.Services{}
+		}
+	}
+	return servicesXML
+}
+
+func promptNodes(r *bufio.Reader, clusterID string, defaultValue xml.Nodes) xml.Nodes {
+	count := promptNodeCount(r, clusterID, defaultValue.Count)
+	const autoSpec = "auto"
+	defaultSpec := autoSpec
+	resources := defaultValue.Resources
+	if resources != nil {
+		defaultSpec = defaultValue.Resources.String()
+	}
+	spec := promptResources(r, clusterID, defaultSpec)
+	if spec == autoSpec {
+		resources = nil
+	} else {
+		r, err := xml.ParseResources(spec)
+		if err != nil {
+			fatalErr(err) // Should not happen as resources have already been validated
+			return xml.Nodes{}
+		}
+		resources = &r
+	}
+	return xml.Nodes{Count: count, Resources: resources}
+}
+
+func promptNodeCount(r *bufio.Reader, clusterID string, nodeCount string) string {
+	fmt.Fprintln(stdout, color.Cyan("\n> Node count: "+clusterID+" cluster"))
+	fmt.Fprintf(stdout, "Documentation: %s\n", color.Green("https://cloud.vespa.ai/en/reference/services"))
+	fmt.Fprintf(stdout, "Example: %s\nExample: %s\n\n", color.Yellow("4"), color.Yellow("[2,8]"))
+	validator := func(input string) error {
+		_, _, err := xml.ParseNodeCount(input)
+		return err
+	}
+	return prompt(r, fmt.Sprintf("How many nodes should the %s cluster have?", color.Cyan(clusterID)), nodeCount, validator)
+}
+
+func promptResources(r *bufio.Reader, clusterID string, resources string) string {
+	fmt.Fprintln(stdout, color.Cyan("\n> Node resources: "+clusterID+" cluster"))
+	fmt.Fprintf(stdout, "Documentation: %s\n", color.Green("https://cloud.vespa.ai/en/reference/services"))
+	fmt.Fprintf(stdout, "Example: %s\nExample: %s\n\n", color.Yellow("auto"), color.Yellow("vcpu=4,memory=8Gb,disk=100Gb"))
+	validator := func(input string) error {
+		if input == "auto" {
+			return nil
+		}
+		_, err := xml.ParseResources(input)
+		return err
+	}
+	return prompt(r, fmt.Sprintf("Which resources should each node in the %s cluster have?", color.Cyan(clusterID)), resources, validator)
+}
+
+func readDeploymentXML(pkg vespa.ApplicationPackage) (xml.Deployment, error) {
+	f, err := os.Open(filepath.Join(pkg.Path, "deployment.xml"))
+	if errors.Is(err, os.ErrNotExist) {
+		// Return a default value if there is no current deployment.xml
+		return xml.DefaultDeployment, nil
+	} else if err != nil {
+		return xml.Deployment{}, err
+	}
+	defer f.Close()
+	return xml.ReadDeployment(f)
+}
+
+func readServicesXML(pkg vespa.ApplicationPackage) (xml.Services, error) {
+	f, err := os.Open(filepath.Join(pkg.Path, "services.xml"))
+	if err != nil {
+		return xml.Services{}, err
+	}
+	defer f.Close()
+	return xml.ReadServices(f)
+}
+
+func prompt(r *bufio.Reader, question, defaultAnswer string, validator func(input string) error) string {
+	var input string
+	for input == "" {
+		fmt.Fprint(stdout, question)
+		if defaultAnswer != "" {
+			fmt.Fprint(stdout, " [", color.Yellow(defaultAnswer), "]")
+		}
+		fmt.Fprint(stdout, " ")
+
+		var err error
+		input, err = r.ReadString('\n')
+		if err != nil {
+			fatalErr(err)
+			return ""
+		}
+		input = strings.TrimSpace(input)
+		if input == "" {
+			input = defaultAnswer
+		}
+
+		if err := validator(input); err != nil {
+			printErr(err)
+			fmt.Fprintln(stderr)
+			input = ""
+		}
+	}
+	return input
+}
diff --git a/client/go/cmd/prod_test.go b/client/go/cmd/prod_test.go
new file mode 100644
index 00000000000..5abb6d24577
--- /dev/null
+++ b/client/go/cmd/prod_test.go
@@ -0,0 +1,118 @@
+package cmd
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/vespa-engine/vespa/client/go/util"
+)
+
+func TestProdInit(t *testing.T) {
+	homeDir := filepath.Join(t.TempDir(), ".vespa")
+	pkgDir := filepath.Join(t.TempDir(), "app")
+	createApplication(t, pkgDir)
+
+	answers := []string{
+		// Regions
+		"invalid input",
+		"aws-us-west-2a,aws-eu-west-1a",
+
+		// Node count: qrs
+		"invalid input",
+		"4",
+
+		// Node resources: qrs
+		"invalid input",
+		"auto",
+
+		// Node count: music
+		"invalid input",
+		"6",
+
+		// Node resources: music
+		"invalid input",
+		"vcpu=16,memory=64Gb,disk=100Gb",
+	}
+	var buf bytes.Buffer
+	buf.WriteString(strings.Join(answers, "\n") + "\n")
+	execute(command{stdin: &buf, homeDir: homeDir, args: []string{"prod", "init", pkgDir}}, t, nil)
+
+	// Verify contents
+	deploymentPath := filepath.Join(pkgDir, "src", "main", "application", "deployment.xml")
+	deploymentXML := readFileString(t, deploymentPath)
+	assert.Contains(t, deploymentXML, `<region active="true">aws-us-west-2a</region>`)
+	assert.Contains(t, deploymentXML, `<region active="true">aws-eu-west-1a</region>`)
+
+	servicesPath := filepath.Join(pkgDir, "src", "main", "application", "services.xml")
+	servicesXML := readFileString(t, servicesPath)
+	containerFragment := `<container id="qrs" version="1.0">
+    <document-api></document-api>
+    <search></search>
+    <nodes count="4"></nodes>
+  </container>`
+	assert.Contains(t, servicesXML, containerFragment)
+	contentFragment := `<content id="music" version="1.0">
+    <redundancy>2</redundancy>
+    <documents>
+      <document type="music" mode="index"></document>
+    </documents>
+    <nodes count="6">
+      <resources vcpu="16" memory="64Gb" disk="100Gb"></resources>
+    </nodes>
+  </content>`
+	assert.Contains(t, servicesXML, contentFragment)
+
+	// Backups are created
+	assert.True(t, util.PathExists(deploymentPath+".1.bak"))
+	assert.True(t, util.PathExists(servicesPath+".1.bak"))
+}
+
+func readFileString(t *testing.T, filename string) string {
+	content, err := ioutil.ReadFile(filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(content)
+}
+
+func createApplication(t *testing.T, pkgDir string) {
+	appDir := filepath.Join(pkgDir, "src", "main", "application")
+	if err := os.MkdirAll(appDir, 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	deploymentXML := `<deployment version="1.0">
+  <prod>
+    <region active="true">aws-us-east-1c</region>
+  </prod>
+</deployment>`
+	if err := ioutil.WriteFile(filepath.Join(appDir, "deployment.xml"), []byte(deploymentXML), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	servicesXML := `<services version="1.0" xmlns:deploy="vespa" xmlns:preprocess="properties">
+  <container id="qrs" version="1.0">
+    <document-api/>
+    <search/>
+    <nodes count="2">
+      <resources vcpu="4" memory="8Gb" disk="100Gb"/>
+    </nodes>
+  </container>
+  <content id="music" version="1.0">
+    <redundancy>2</redundancy>
+    <documents>
+      <document type="music" mode="index"></document>
+    </documents>
+    <nodes count="4"></nodes>
+  </content>
+</services>`
+
+	if err := ioutil.WriteFile(filepath.Join(appDir, "services.xml"), []byte(servicesXML), 0644); err != nil {
+		t.Fatal(err)
+	}
+}
diff --git a/client/go/cmd/root.go b/client/go/cmd/root.go
index ab6c19a665b..5aae55ab6e4 100644
--- a/client/go/cmd/root.go
+++ b/client/go/cmd/root.go
@@ -6,6 +6,7 @@ package cmd
 
 import (
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"os"
@@ -37,6 +38,7 @@ Vespa documentation: https://docs.vespa.ai`,
 	waitSecsArg    int
 	colorArg       string
 	quietArg       bool
+	stdin          io.ReadWriter = os.Stdin
 
 	color  = aurora.NewAurora(false)
 	stdout = colorable.NewColorableStdout()
diff --git a/client/go/vespa/xml/config.go b/client/go/vespa/xml/config.go
index c9b65822e7f..a30db238afb 100644
--- a/client/go/vespa/xml/config.go
+++ b/client/go/vespa/xml/config.go
@@ -194,7 +194,10 @@ func ParseNodeCount(s string) (int, int, error) {
 }
 
 // ValidProdRegion returns whether string s is a valid production region.
-func ValidProdRegion(s string) bool {
+func ValidProdRegion(s string, system string) bool {
+	if system == "publiccd" {
+		return s == "aws-us-east-1c"
+	}
 	switch s {
 	case "aws-us-east-1c", "aws-us-west-2a",
 		"aws-eu-west-1a", "aws-ap-northeast-1a":