Implement vespa prod submit

10 files changed, 284 insertions, 41 deletions

diff --git a/client/go/cmd/command_tester.go b/client/go/cmd/command_tester.go
index 201adb8cab7..2d2de6a201c 100644
--- a/client/go/cmd/command_tester.go
+++ b/client/go/cmd/command_tester.go
@@ -58,6 +58,8 @@ func execute(cmd command, t *testing.T, client *mockHttpClient) (string, string)
 	os.Setenv("VESPA_CLI_CACHE_DIR", cmd.cacheDir)
 
 	// Reset flags to their default value - persistent flags in Cobra persists over tests
+	// TODO: Due to the bad design of viper, the only proper fix is to get rid of global state by moving each command to
+	// their own sub-package
 	rootCmd.Flags().VisitAll(resetFlag)
 	documentCmd.Flags().VisitAll(resetFlag)
 
diff --git a/client/go/cmd/deploy.go b/client/go/cmd/deploy.go
index 7c9c3d8cc02..034dac2e67b 100644
--- a/client/go/cmd/deploy.go
+++ b/client/go/cmd/deploy.go
@@ -62,20 +62,7 @@ $ vespa deploy -t cloud -z perf.aws-us-east-1c`,
 			return
 		}
 		target := getTarget()
-		opts := vespa.DeploymentOpts{ApplicationPackage: pkg, Target: target}
-		if opts.IsCloud() {
-			deployment := deploymentFromArgs()
-			if !opts.ApplicationPackage.HasCertificate() {
-				fatalErrHint(fmt.Errorf("Missing certificate in application package"), "Applications in Vespa Cloud require a certificate", "Try 'vespa cert'")
-				return
-			}
-			opts.APIKey, err = cfg.ReadAPIKey(deployment.Application.Tenant)
-			if err != nil {
-				fatalErrHint(err, "Deployment to cloud requires an API key. Try 'vespa api-key'")
-				return
-			}
-			opts.Deployment = deployment
-		}
+		opts := getDeploymentOpts(cfg, pkg, target)
 		if sessionOrRunID, err := vespa.Deploy(opts); err == nil {
 			if opts.IsCloud() {
 				printSuccess("Triggered deployment of ", color.Cyan(pkg.Path), " with run ID ", color.Cyan(sessionOrRunID))
diff --git a/client/go/cmd/helpers.go b/client/go/cmd/helpers.go
index b5525cf11fe..4768290e33e 100644
--- a/client/go/cmd/helpers.go
+++ b/client/go/cmd/helpers.go
@@ -30,7 +30,9 @@ func fatalErr(err error, msg ...interface{}) {
 }
 
 func printErrHint(err error, hints ...string) {
-	printErr(nil, err.Error())
+	if err != nil {
+		printErr(nil, err.Error())
+	}
 	for _, hint := range hints {
 		fmt.Fprintln(stderr, color.Cyan("Hint:"), hint)
 	}
@@ -140,9 +142,10 @@ func getService(service string, sessionOrRunID int64) *vespa.Service {
 	return s
 }
 
+func getSystem() string { return os.Getenv("VESPA_CLI_CLOUD_SYSTEM") }
+
 func getConsoleURL() string {
-	system := os.Getenv("VESPA_CLI_CLOUD_SYSTEM")
-	if system == "publiccd" {
+	if getSystem() == "publiccd" {
 		return "https://console-cd.vespa.oath.cloud"
 	}
 	return "https://console.vespa.oath.cloud"
@@ -150,8 +153,7 @@ func getConsoleURL() string {
 }
 
 func getApiURL() string {
-	system := os.Getenv("VESPA_CLI_CLOUD_SYSTEM")
-	if system == "publiccd" {
+	if getSystem() == "publiccd" {
 		return "https://api.vespa-external-cd.aws.oath.cloud:4443"
 	}
 	return "https://api.vespa-external.aws.oath.cloud:4443"
@@ -221,3 +223,22 @@ func waitForService(service string, sessionOrRunID int64) {
 		fatalErr(err, s.Description(), " at ", color.Cyan(s.BaseURL), " is ", color.Red("not ready"))
 	}
 }
+
+func getDeploymentOpts(cfg *Config, pkg vespa.ApplicationPackage, target vespa.Target) vespa.DeploymentOpts {
+	opts := vespa.DeploymentOpts{ApplicationPackage: pkg, Target: target}
+	if opts.IsCloud() {
+		deployment := deploymentFromArgs()
+		if !opts.ApplicationPackage.HasCertificate() {
+			fatalErrHint(fmt.Errorf("Missing certificate in application package"), "Applications in Vespa Cloud require a certificate", "Try 'vespa cert'")
+			return opts
+		}
+		var err error
+		opts.APIKey, err = cfg.ReadAPIKey(deployment.Application.Tenant)
+		if err != nil {
+			fatalErrHint(err, "Deployment to cloud requires an API key. Try 'vespa api-key'")
+			return opts
+		}
+		opts.Deployment = deployment
+	}
+	return opts
+}
diff --git a/client/go/cmd/prod.go b/client/go/cmd/prod.go
index 15d9edc18a7..382ede0fae8 100644
--- a/client/go/cmd/prod.go
+++ b/client/go/cmd/prod.go
@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"os"
 	"path/filepath"
 	"strings"
@@ -20,6 +21,7 @@ import (
 func init() {
 	rootCmd.AddCommand(prodCmd)
 	prodCmd.AddCommand(prodInitCmd)
+	prodCmd.AddCommand(prodSubmitCmd)
 }
 
 var prodCmd = &cobra.Command{
@@ -60,7 +62,7 @@ https://cloud.vespa.ai/en/reference/deployment`,
 		}
 		if pkg.IsZip() {
 			fatalErrHint(fmt.Errorf("Cannot modify compressed application package %s", pkg.Path),
-				"Try running 'mvn clean' and re-running this command")
+				"Try running 'mvn clean' and run this command again")
 			return
 		}
 
@@ -95,6 +97,69 @@ https://cloud.vespa.ai/en/reference/deployment`,
 	},
 }
 
+var prodSubmitCmd = &cobra.Command{
+	Use:   "submit",
+	Short: "Submit your application for production deployment",
+	Long: `Submit your application for production deployment.
+
+This commands uploads your application package to Vespa Cloud and deploys it to
+the production zones specified in deployment.xml.
+
+Nodes are allocated to your application according to resources specified in
+services.xml.
+
+While submitting an application from a local development environment is
+supported, it's strongly recommended that production deployments are performed
+by a continuous build system.
+
+For more information about production deployments in Vespa Cloud see:
+https://cloud.vespa.ai/en/getting-to-production
+https://cloud.vespa.ai/en/automated-deployments`,
+	DisableAutoGenTag: true,
+	Example: `$ mvn package
+$ vespa prod submit`,
+	Run: func(cmd *cobra.Command, args []string) {
+		target := getTarget()
+		if target.Type() != "cloud" {
+			fatalErr(fmt.Errorf("%s target cannot deploy to Vespa Cloud", target.Type()))
+			return
+		}
+		appSource := applicationSource(args)
+		pkg, err := vespa.FindApplicationPackage(appSource, true)
+		if err != nil {
+			fatalErr(err)
+			return
+		}
+		cfg, err := LoadConfig()
+		if err != nil {
+			fatalErr(err, "Could not load config")
+			return
+		}
+		if !pkg.HasDeployment() {
+			fatalErrHint(fmt.Errorf("No deployment.xml found"), "Try creating one with vespa prod init")
+			return
+		}
+		if !pkg.IsJava() {
+			// TODO: Loosen this requirement when we start supporting applications with Java in production
+			fatalErrHint(fmt.Errorf("No jar files found in %s", pkg.Path), "Only applications containing Java components are currently supported")
+			return
+		}
+		isCI := os.Getenv("CI") != ""
+		if !isCI {
+			fmt.Fprintln(stderr, color.Yellow("Warning:"), "Submitting from a non-CI environment is discouraged")
+			printErrHint(nil, "See https://cloud.vespa.ai/en/getting-to-production for best practices")
+		}
+		opts := getDeploymentOpts(cfg, pkg, target)
+		if err := vespa.Submit(opts); err != nil {
+			fatalErr(err, "Could not submit application for deployment")
+		} else {
+			printSuccess("Submitted ", color.Cyan(pkg.Path), " for deployment")
+			log.Printf("See %s for deployment progress\n", color.Cyan(fmt.Sprintf("%s/tenant/%s/application/%s/prod/deployment",
+				getConsoleURL(), opts.Deployment.Application.Tenant, opts.Deployment.Application.Application)))
+		}
+	},
+}
+
 func writeWithBackup(pkg vespa.ApplicationPackage, filename, contents string) error {
 	dst := filepath.Join(pkg.Path, filename)
 	if util.PathExists(dst) {
@@ -133,6 +198,13 @@ func updateRegions(r *bufio.Reader, deploymentXML xml.Deployment) xml.Deployment
 	if err := deploymentXML.Replace("prod", "region", regionElements); err != nil {
 		fatalErr(err, "Could not update region elements in deployment.xml")
 	}
+	// TODO: Some sample apps come with production <test> elements, but not necessarily working production tests, we
+	//       therefore remove <test> elements here.
+	//       This can be improved by supporting <test> elements in xml package and allow specifying testing as part of
+	//       region prompt, e.g. region1;test,region2
+	if err := deploymentXML.Replace("prod", "test", nil); err != nil {
+		fatalErr(err, "Could not remove test elements in deployment.xml")
+	}
 	return deploymentXML
 }
 
@@ -152,7 +224,7 @@ func promptRegions(r *bufio.Reader, deploymentXML xml.Deployment) string {
 	validator := func(input string) error {
 		regions := strings.Split(input, ",")
 		for _, r := range regions {
-			if !xml.ValidProdRegion(r) {
+			if !xml.IsProdRegion(r, getSystem()) {
 				return fmt.Errorf("invalid region %s", r)
 			}
 		}
diff --git a/client/go/cmd/prod_test.go b/client/go/cmd/prod_test.go
index 5abb6d24577..4ce6112122a 100644
--- a/client/go/cmd/prod_test.go
+++ b/client/go/cmd/prod_test.go
@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"bytes"
+	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -82,6 +83,7 @@ func readFileString(t *testing.T, filename string) string {
 
 func createApplication(t *testing.T, pkgDir string) {
 	appDir := filepath.Join(pkgDir, "src", "main", "application")
+	targetDir := filepath.Join(pkgDir, "target")
 	if err := os.MkdirAll(appDir, 0755); err != nil {
 		t.Fatal(err)
 	}
@@ -115,4 +117,50 @@ func createApplication(t *testing.T, pkgDir string) {
 	if err := ioutil.WriteFile(filepath.Join(appDir, "services.xml"), []byte(servicesXML), 0644); err != nil {
 		t.Fatal(err)
 	}
+	if err := os.MkdirAll(targetDir, 0755); err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(filepath.Join(pkgDir, "pom.xml"), []byte(""), 0644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestProdSubmit(t *testing.T) {
+	homeDir := filepath.Join(t.TempDir(), ".vespa")
+	pkgDir := filepath.Join(t.TempDir(), "app")
+	createApplication(t, pkgDir)
+
+	httpClient := &mockHttpClient{}
+	httpClient.NextResponse(200, `ok`)
+	execute(command{homeDir: homeDir, args: []string{"config", "set", "application", "t1.a1.i1"}}, t, httpClient)
+	execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "cloud"}}, t, httpClient)
+	execute(command{homeDir: homeDir, args: []string{"api-key"}}, t, httpClient)
+	execute(command{homeDir: homeDir, args: []string{"cert", pkgDir}}, t, httpClient)
+
+	// Copy an application package pre-assambled with mvn package
+	testAppDir := filepath.Join("testdata", "applications", "withDeployment", "target")
+	zipFile := filepath.Join(testAppDir, "application.zip")
+	copyFile(t, filepath.Join(pkgDir, "target", "application.zip"), zipFile)
+	testZipFile := filepath.Join(testAppDir, "application-test.zip")
+	copyFile(t, filepath.Join(pkgDir, "target", "application-test.zip"), testZipFile)
+
+	out, _ := execute(command{homeDir: homeDir, args: []string{"prod", "submit", pkgDir}}, t, httpClient)
+	assert.Contains(t, out, "Success: Submitted")
+	assert.Contains(t, out, "See https://console.vespa.oath.cloud/tenant/t1/application/a1/prod/deployment for deployment progress")
+}
+
+func copyFile(t *testing.T, dstFilename, srcFilename string) {
+	dst, err := os.Create(dstFilename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer dst.Close()
+	src, err := os.Open(srcFilename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer src.Close()
+	if _, err := io.Copy(dst, src); err != nil {
+		t.Fatal(err)
+	}
 }
diff --git a/client/go/cmd/query_test.go b/client/go/cmd/query_test.go
index 81dc03766be..ec6c3063906 100644
--- a/client/go/cmd/query_test.go
+++ b/client/go/cmd/query_test.go
@@ -45,12 +45,12 @@ func TestServerError(t *testing.T) {
 
 func assertQuery(t *testing.T, expectedQuery string, query ...string) {
 	client := &mockHttpClient{}
-	queryURL := queryServiceURL(client)
 	client.NextResponse(200, "{\"query\":\"result\"}")
 	assert.Equal(t,
 		"{\n    \"query\": \"result\"\n}\n",
 		executeCommand(t, client, []string{"query"}, query),
 		"query output")
+	queryURL := queryServiceURL(client)
 	assert.Equal(t, queryURL+"/search/"+expectedQuery, client.lastRequest.URL.String())
 }
 
diff --git a/client/go/cmd/testdata/applications/withDeployment/target/application-test.zip b/client/go/cmd/testdata/applications/withDeployment/target/application-test.zip
new file mode 100644
index 00000000000..8a1707b9cee
--- /dev/null
+++ b/client/go/cmd/testdata/applications/withDeployment/target/application-test.zip
diff --git a/client/go/cmd/testdata/applications/withDeployment/target/application.zip b/client/go/cmd/testdata/applications/withDeployment/target/application.zip
new file mode 100644
index 00000000000..da23c2ff437
--- /dev/null
+++ b/client/go/cmd/testdata/applications/withDeployment/target/application.zip
diff --git a/client/go/vespa/deploy.go b/client/go/vespa/deploy.go
index 908b3772b70..3718c7d813a 100644
--- a/client/go/vespa/deploy.go
+++ b/client/go/vespa/deploy.go
@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"mime/multipart"
 	"net/http"
 	"net/url"
 	"os"
@@ -49,7 +50,8 @@ type DeploymentOpts struct {
 }
 
 type ApplicationPackage struct {
-	Path string
+	Path     string
+	TestPath string
 }
 
 func (a ApplicationID) String() string {
@@ -79,6 +81,15 @@ func (d *DeploymentOpts) url(path string) (*url.URL, error) {
 }
 
 func (ap *ApplicationPackage) HasCertificate() bool {
+	return ap.hasFile(filepath.Join("security", "clients.pem"), "security/clients.pem")
+}
+
+func (ap *ApplicationPackage) HasDeployment() bool { return ap.hasFile("deployment.xml", "") }
+
+func (ap *ApplicationPackage) hasFile(filename, zipName string) bool {
+	if zipName == "" {
+		zipName = filename
+	}
 	if ap.IsZip() {
 		r, err := zip.OpenReader(ap.Path)
 		if err != nil {
@@ -86,35 +97,58 @@ func (ap *ApplicationPackage) HasCertificate() bool {
 		}
 		defer r.Close()
 		for _, f := range r.File {
-			if f.Name == "security/clients.pem" {
+			if f.Name == zipName {
 				return true
 			}
 		}
 		return false
 	}
-	return util.PathExists(filepath.Join(ap.Path, "security", "clients.pem"))
+	return util.PathExists(filepath.Join(ap.Path, filename))
 }
 
 func (ap *ApplicationPackage) IsZip() bool { return isZip(ap.Path) }
 
-func (ap *ApplicationPackage) zipReader() (io.ReadCloser, error) {
+func (ap *ApplicationPackage) IsJava() bool {
+	if ap.IsZip() {
+		r, err := zip.OpenReader(ap.Path)
+		if err != nil {
+			return false
+		}
+		defer r.Close()
+		for _, f := range r.File {
+			if filepath.Ext(f.Name) == ".jar" {
+				return true
+			}
+		}
+		return false
+	}
+	return util.PathExists(filepath.Join(ap.Path, "pom.xml"))
+}
+
+func (ap *ApplicationPackage) zipReader(test bool) (io.ReadCloser, error) {
 	zipFile := ap.Path
+	if test {
+		zipFile = ap.TestPath
+	}
 	if !ap.IsZip() {
-		tempZip, error := ioutil.TempFile("", "application.zip")
-		if error != nil {
-			return nil, fmt.Errorf("Could not create a temporary zip file for the application package: %w", error)
+		tempZip, err := ioutil.TempFile("", "vespa")
+		if err != nil {
+			return nil, fmt.Errorf("Could not create a temporary zip file for the application package: %w", err)
 		}
+		defer func() {
+			tempZip.Close()
+			os.Remove(tempZip.Name())
+		}()
 		if err := zipDir(ap.Path, tempZip.Name()); err != nil {
 			return nil, err
 		}
-		defer os.Remove(tempZip.Name())
 		zipFile = tempZip.Name()
 	}
-	r, err := os.Open(zipFile)
+	f, err := os.Open(zipFile)
 	if err != nil {
 		return nil, fmt.Errorf("Could not open application package at %s: %w", ap.Path, err)
 	}
-	return r, nil
+	return f, nil
 }
 
 // FindApplicationPackage finds the path to an application package from the zip file or directory zipOrDir.
@@ -125,7 +159,8 @@ func FindApplicationPackage(zipOrDir string, requirePackaging bool) (Application
 	if util.PathExists(filepath.Join(zipOrDir, "pom.xml")) {
 		zip := filepath.Join(zipOrDir, "target", "application.zip")
 		if util.PathExists(zip) {
-			return ApplicationPackage{Path: zip}, nil
+			testZip := filepath.Join(zipOrDir, "target", "application-test.zip")
+			return ApplicationPackage{Path: zip, TestPath: testZip}, nil
 		}
 		if requirePackaging {
 			return ApplicationPackage{}, errors.New("pom.xml exists but no target/application.zip. Run mvn package first")
@@ -214,11 +249,8 @@ func Activate(sessionID int64, deployment DeploymentOpts) error {
 func Deploy(opts DeploymentOpts) (int64, error) {
 	path := "/application/v2/tenant/default/prepareandactivate"
 	if opts.IsCloud() {
-		if !opts.ApplicationPackage.HasCertificate() {
-			return 0, fmt.Errorf("%s: missing certificate in package", opts)
-		}
-		if opts.APIKey == nil {
-			return 0, fmt.Errorf("%s: missing api key", opts.String())
+		if err := checkDeploymentOpts(opts); err != nil {
+			return 0, err
 		}
 		if opts.Deployment.Zone.Environment == "" || opts.Deployment.Zone.Region == "" {
 			return 0, fmt.Errorf("%s: missing zone", opts)
@@ -237,8 +269,89 @@ func Deploy(opts DeploymentOpts) (int64, error) {
 	return uploadApplicationPackage(u, opts)
 }
 
+func copyToPart(dst *multipart.Writer, src io.Reader, fieldname, filename string) error {
+	var part io.Writer
+	var err error
+	if filename == "" {
+		part, err = dst.CreateFormField(fieldname)
+	} else {
+		part, err = dst.CreateFormFile(fieldname, filename)
+	}
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(part, src); err != nil {
+		return err
+	}
+	return nil
+}
+
+func Submit(opts DeploymentOpts) error {
+	if !opts.IsCloud() {
+		return fmt.Errorf("%s: submit is unsupported", opts)
+	}
+	if err := checkDeploymentOpts(opts); err != nil {
+		return err
+	}
+	path := fmt.Sprintf("/application/v4/tenant/%s/application/%s/submit", opts.Deployment.Application.Tenant, opts.Deployment.Application.Application)
+	u, err := opts.url(path)
+	if err != nil {
+		return err
+	}
+	var body bytes.Buffer
+	writer := multipart.NewWriter(&body)
+	if err := copyToPart(writer, strings.NewReader("{}"), "submitOptions", ""); err != nil {
+		return err
+	}
+	applicationZip, err := opts.ApplicationPackage.zipReader(false)
+	if err != nil {
+		return err
+	}
+	if err := copyToPart(writer, applicationZip, "applicationZip", "application.zip"); err != nil {
+		return err
+	}
+	testApplicationZip, err := opts.ApplicationPackage.zipReader(true)
+	if err != nil {
+		return err
+	}
+	if err := copyToPart(writer, testApplicationZip, "applicationTestZip", "application-test.zip"); err != nil {
+		return err
+	}
+	if err := writer.Close(); err != nil {
+		return err
+	}
+	request := &http.Request{
+		URL:    u,
+		Method: "POST",
+		Body:   ioutil.NopCloser(&body),
+		Header: make(http.Header),
+	}
+	request.Header.Set("Content-Type", writer.FormDataContentType())
+	signer := NewRequestSigner(opts.Deployment.Application.SerializedForm(), opts.APIKey)
+	if err := signer.SignRequest(request); err != nil {
+		return err
+	}
+	serviceDescription := "Submit service"
+	response, err := util.HttpDo(request, time.Minute*10, serviceDescription)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+	return checkResponse(request, response, serviceDescription)
+}
+
+func checkDeploymentOpts(opts DeploymentOpts) error {
+	if !opts.ApplicationPackage.HasCertificate() {
+		return fmt.Errorf("%s: missing certificate in package", opts)
+	}
+	if opts.APIKey == nil {
+		return fmt.Errorf("%s: missing api key", opts.String())
+	}
+	return nil
+}
+
 func uploadApplicationPackage(url *url.URL, opts DeploymentOpts) (int64, error) {
-	zipReader, err := opts.ApplicationPackage.zipReader()
+	zipReader, err := opts.ApplicationPackage.zipReader(false)
 	if err != nil {
 		return 0, err
 	}
diff --git a/client/go/vespa/xml/config.go b/client/go/vespa/xml/config.go
index a30db238afb..e900b50cbb0 100644
--- a/client/go/vespa/xml/config.go
+++ b/client/go/vespa/xml/config.go
@@ -193,8 +193,8 @@ func ParseNodeCount(s string) (int, int, error) {
 	return 0, 0, parseErr
 }
 
-// ValidProdRegion returns whether string s is a valid production region.
-func ValidProdRegion(s string, system string) bool {
+// IsProdRegion returns whether string s is a valid production region.
+func IsProdRegion(s string, system string) bool {
 	if system == "publiccd" {
 		return s == "aws-us-east-1c"
 	}