Refactor deploy to support cloud

8 files changed, 294 insertions, 116 deletions

diff --git a/client/go/cmd/activate.go b/client/go/cmd/activate.go
deleted file mode 100644
index 70aed151a6d..00000000000
--- a/client/go/cmd/activate.go
+++ /dev/null
@@ -1,25 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-// vespa activate command
-// Author: bratseth
-
-package cmd
-
-import (
-	"github.com/spf13/cobra"
-)
-
-func init() {
-	rootCmd.AddCommand(activateCmd)
-	addTargetFlag(activateCmd)
-}
-
-// TODO: Implement and test
-
-var activateCmd = &cobra.Command{
-	Use:   "activate",
-	Short: "Activates (deploys) the previously prepared application package",
-	Long:  `TODO`,
-	Run: func(cmd *cobra.Command, args []string) {
-		deploy(true, nil)
-	},
-}
diff --git a/client/go/cmd/cert.go b/client/go/cmd/cert.go
index 88d489fd229..f76a7041a82 100644
--- a/client/go/cmd/cert.go
+++ b/client/go/cmd/cert.go
@@ -15,16 +15,19 @@ import (
 var overwriteCertificate bool
 
 func init() {
-	certCmd.Flags().BoolVarP(&overwriteCertificate, "force", "f", false, "Force overwrite of existing certificate and private key")
 	rootCmd.AddCommand(certCmd)
+	certCmd.Flags().BoolVarP(&overwriteCertificate, "force", "f", false, "Force overwrite of existing certificate and private key")
+	certCmd.PersistentFlags().StringVarP(&applicationArg, applicationFlag, "a", "", "The application owning this certificate")
+	certCmd.MarkPersistentFlagRequired(applicationFlag)
 }
 
 var certCmd = &cobra.Command{
 	Use:   "cert",
 	Short: "Creates a new private key and self-signed certificate",
-	Long: `Applications in Vespa Cloud are required to secure their data plane with mutual TLS. ` +
-		`This command creates a self-signed certificate suitable for development purposes. ` +
-		`See https://cloud.vespa.ai/en/security-model for more information on the Vespa Cloud security model.`,
+	Long: "Applications in Vespa Cloud are required to secure their data plane with mutual TLS.\n\n" +
+		"This command creates a self-signed certificate suitable for development purposes.\n" +
+		"See https://cloud.vespa.ai/en/security-model for more information on the Vespa\n" +
+		"Cloud security model.",
 	Run: func(cmd *cobra.Command, args []string) {
 		var path string
 		if len(args) > 0 {
@@ -35,23 +38,28 @@ var certCmd = &cobra.Command{
 			fatalIfErr(err)
 		}
 
-		pkg, err := vespa.FindApplicationPackage(path)
+		app, err := vespa.ApplicationFromString(applicationArg)
+		fatalIfErr(err)
+
+		pkg, err := vespa.ApplicationPackageFrom(path)
 		fatalIfErr(err)
 		if pkg.HasCertificate() && !overwriteCertificate {
 			log.Print("Certificate already exists. Use -f option to recreate")
 			return
 		}
 
-		// TODO: Consider writing key pair inside ~/.vespa/<app-name>/ instead so that vespa document commands can easily
-		// locate key pair
+		configDir, err := configDir(app.String())
+		fatalIfErr(err)
 		securityDir := filepath.Join(pkg.Path, "security")
-		privateKeyFile := filepath.Join(path, "data-plane-private-key.pem")
-		certificateFile := filepath.Join(path, "data-plane-public-cert.pem")
 		pkgCertificateFile := filepath.Join(securityDir, "clients.pem")
+		privateKeyFile := filepath.Join(configDir, "data-plane-private-key.pem")
+		certificateFile := filepath.Join(configDir, "data-plane-public-cert.pem")
 
 		keyPair, err := vespa.CreateKeyPair()
 		fatalIfErr(err)
 
+		err = os.MkdirAll(configDir, 0755)
+		fatalIfErr(err)
 		err = os.MkdirAll(securityDir, 0755)
 		fatalIfErr(err)
 		err = keyPair.WriteCertificateFile(pkgCertificateFile, overwriteCertificate)
@@ -61,7 +69,6 @@ var certCmd = &cobra.Command{
 		err = keyPair.WritePrivateKeyFile(privateKeyFile, overwriteCertificate)
 		fatalIfErr(err)
 
-		// TODO: Just use log package, which has Printf
 		log.Printf("Certificate written to %s", color.Green(pkgCertificateFile))
 		log.Printf("Certificate written to %s", color.Green(certificateFile))
 		log.Printf("Private key written to %s", color.Green(privateKeyFile))
diff --git a/client/go/cmd/deploy.go b/client/go/cmd/deploy.go
index ed7f7264fcc..76148f41291 100644
--- a/client/go/cmd/deploy.go
+++ b/client/go/cmd/deploy.go
@@ -6,14 +6,33 @@ package cmd
 
 import (
 	"log"
+	"os"
+	"path/filepath"
 
 	"github.com/spf13/cobra"
 	"github.com/vespa-engine/vespa/vespa"
 )
 
+const (
+	zoneFlag        = "zone"
+	applicationFlag = "application"
+)
+
+var (
+	zoneArg        string
+	applicationArg string
+)
+
 func init() {
 	rootCmd.AddCommand(deployCmd)
+	rootCmd.AddCommand(prepareCmd)
+	rootCmd.AddCommand(activateCmd)
 	addTargetFlag(deployCmd)
+	addTargetFlag(prepareCmd)
+	addTargetFlag(activateCmd)
+
+	deployCmd.PersistentFlags().StringVarP(&zoneArg, zoneFlag, "z", "dev.aws-us-east-1c", "The zone to use for deployment")
+	deployCmd.PersistentFlags().StringVarP(&applicationArg, applicationFlag, "a", "", "The application name to use for deployment")
 }
 
 var deployCmd = &cobra.Command{
@@ -21,19 +40,81 @@ var deployCmd = &cobra.Command{
 	Short: "Deploys (prepares and activates) an application package",
 	Long:  `TODO`,
 	Run: func(cmd *cobra.Command, args []string) {
-		deploy(false, args)
+		d := vespa.Deployment{
+			ApplicationSource: applicationSource(args),
+			TargetType:        targetArg,
+			TargetURL:         deployTarget(),
+		}
+		if d.IsCloud() {
+			var err error
+			d.Zone, err = vespa.ZoneFromString(zoneArg)
+			if err != nil {
+				log.Fatal(err)
+			}
+			d.Application, err = vespa.ApplicationFromString(applicationArg)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			d.KeyPair, err = loadApplicationKeyPair(applicationArg)
+			if err != nil {
+				log.Fatal(err)
+			}
+		}
+		resolvedSrc, err := vespa.Deploy(d)
+		if err == nil {
+			log.Printf("Deployed %s successfully", color.Cyan(resolvedSrc))
+		} else {
+			log.Print(err)
+		}
+	},
+}
+
+var prepareCmd = &cobra.Command{
+	Use:   "prepare",
+	Short: "Prepares an application package for activation",
+	Long:  `TODO`,
+	Run: func(cmd *cobra.Command, args []string) {
+		resolvedSrc, err := vespa.Prepare(vespa.Deployment{ApplicationSource: applicationSource(args)})
+		if err == nil {
+			log.Printf("Prepared %s successfully", color.Cyan(resolvedSrc))
+		} else {
+			log.Print(color.Red(err))
+		}
+	},
+}
+
+var activateCmd = &cobra.Command{
+	Use:   "activate",
+	Short: "Activates (deploys) the previously prepared application package",
+	Long:  `TODO`,
+	Run: func(cmd *cobra.Command, args []string) {
+		resolvedSrc, err := vespa.Activate(vespa.Deployment{ApplicationSource: applicationSource(args)})
+		if err == nil {
+			log.Printf("Activated %s successfully", color.Cyan(resolvedSrc))
+		} else {
+			log.Print(color.Red(err))
+		}
 	},
 }
 
-func deploy(prepare bool, args []string) {
-	var application string
+func loadApplicationKeyPair(application string) (vespa.PemKeyPair, error) {
+	configDir, err := configDir(application)
+	if err != nil {
+		return vespa.PemKeyPair{}, err
+	}
+	certificateFile := filepath.Join(configDir, "data-plane-public-cert.pem")
+	privateKeyFile := filepath.Join(configDir, "data-plane-private-key.pem")
+	return vespa.LoadKeyPair(privateKeyFile, certificateFile)
+}
+
+func applicationSource(args []string) string {
 	if len(args) > 0 {
-		application = args[0]
+		return args[0]
 	}
-	path, err := vespa.Deploy(prepare, application, deployTarget())
+	wd, err := os.Getwd()
 	if err != nil {
-		log.Print(color.Red(err))
-	} else {
-		log.Print("Deployed ", color.Green(path), " successfully")
+		log.Fatalf("Could not determine working directory: %s", err)
 	}
+	return wd
 }
diff --git a/client/go/cmd/prepare.go b/client/go/cmd/prepare.go
deleted file mode 100644
index 5f31d0b8e23..00000000000
--- a/client/go/cmd/prepare.go
+++ /dev/null
@@ -1,25 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-// vespa prepare command
-// Author: bratseth
-
-package cmd
-
-import (
-	"github.com/spf13/cobra"
-)
-
-func init() {
-	rootCmd.AddCommand(prepareCmd)
-	addTargetFlag(prepareCmd)
-}
-
-// TODO: Implement and test
-
-var prepareCmd = &cobra.Command{
-	Use:   "prepare",
-	Short: "Prepares an application package for activation",
-	Long:  `TODO`,
-	Run: func(cmd *cobra.Command, args []string) {
-		deploy(true, args)
-	},
-}
diff --git a/client/go/cmd/target.go b/client/go/cmd/target.go
index 79d3d6f32e8..d9acddc4dc9 100644
--- a/client/go/cmd/target.go
+++ b/client/go/cmd/target.go
@@ -11,9 +11,12 @@ import (
 	"github.com/spf13/cobra"
 )
 
-const flagName = "target"
+const (
+	targetFlag = "target"
+	cloudApi   = "https://api.vespa-external.aws.oath.cloud"
+)
 
-var targetArgument string
+var targetArg string
 
 type target struct {
 	deploy   string
@@ -30,8 +33,8 @@ const (
 )
 
 func addTargetFlag(command *cobra.Command) {
-	command.PersistentFlags().StringVarP(&targetArgument, flagName, "t", "local", "The name or URL of the recipient of this command")
-	bindFlagToConfig(flagName, command)
+	command.PersistentFlags().StringVarP(&targetArg, targetFlag, "t", "local", "The name or URL of the recipient of this command")
+	bindFlagToConfig(targetFlag, command)
 }
 
 func deployTarget() string {
@@ -47,7 +50,7 @@ func documentTarget() string {
 }
 
 func getTarget(targetContext context) *target {
-	targetValue, err := getOption(flagName)
+	targetValue, err := getOption(targetFlag)
 	if err != nil {
 		log.Fatalf("a valid target must be specified")
 	}
@@ -80,7 +83,7 @@ func getTarget(targetContext context) *target {
 	}
 
 	if targetValue == "cloud" {
-		panic("cloud target is not implemented")
+		return &target{deploy: cloudApi}
 	}
 
 	log.Printf("Unknown target '%s': Use %s, %s or an URL", color.Red(targetValue), color.Cyan("local"), color.Cyan("cloud"))
diff --git a/client/go/vespa/crypto.go b/client/go/vespa/crypto.go
index b9e41d81576..a410f65c620 100644
--- a/client/go/vespa/crypto.go
+++ b/client/go/vespa/crypto.go
@@ -83,6 +83,23 @@ func CreateKeyPair() (PemKeyPair, error) {
 	return PemKeyPair{Certificate: pemCertificate, PrivateKey: pemPrivateKey}, nil
 }
 
+// LoadKeyPair reads a key pair located in privateKeyFile and certificateFile.
+func LoadKeyPair(privateKeyFile, certificateFile string) (PemKeyPair, error) {
+	var (
+		kp  PemKeyPair
+		err error
+	)
+	kp.PrivateKey, err = os.ReadFile(privateKeyFile)
+	if err != nil {
+		return PemKeyPair{}, err
+	}
+	kp.Certificate, err = os.ReadFile(certificateFile)
+	if err != nil {
+		return PemKeyPair{}, err
+	}
+	return kp, err
+}
+
 func randomSerialNumber() (*big.Int, error) {
 	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
 	return rand.Int(rand.Reader, serialNumberLimit)
diff --git a/client/go/vespa/deploy.go b/client/go/vespa/deploy.go
index 9973d3fd490..037c92b5458 100644
--- a/client/go/vespa/deploy.go
+++ b/client/go/vespa/deploy.go
@@ -20,12 +20,72 @@ import (
 	"github.com/vespa-engine/vespa/util"
 )
 
+type ApplicationID struct {
+	Tenant      string
+	Application string
+	Instance    string
+}
+
+type ZoneID struct {
+	Environment string
+	Region      string
+}
+
+type Deployment struct {
+	ApplicationSource string
+	TargetType        string
+	TargetURL         string
+	Application       ApplicationID
+	Zone              ZoneID
+	KeyPair           PemKeyPair
+	APIKey            []byte
+}
+
 type ApplicationPackage struct {
 	Path string
 }
 
+func (a ApplicationID) String() string {
+	return fmt.Sprintf("%s.%s.%s", a.Tenant, a.Application, a.Instance)
+}
+
+func (d Deployment) String() string {
+	return fmt.Sprintf("deployment of %s to %s target", d.Application, d.TargetType)
+}
+
+func (d *Deployment) IsCloud() bool { return d.TargetType == "cloud" }
+
+func (ap *ApplicationPackage) IsZip() bool { return isZip(ap.Path) }
+
+func (ap *ApplicationPackage) HasCertificate() bool {
+	if ap.IsZip() {
+		return true // TODO: Consider looking inside zip to verify
+	}
+	return util.PathExists(filepath.Join(ap.Path, "security", "clients.pem"))
+}
+
+func (ap *ApplicationPackage) zipReader() (io.ReadCloser, error) {
+	zipFile := ap.Path
+	if !ap.IsZip() {
+		tempZip, error := ioutil.TempFile("", "application.zip")
+		if error != nil {
+			return nil, fmt.Errorf("Could not create a temporary zip file for the application package: %w", error)
+		}
+		if err := zipDir(ap.Path, tempZip.Name()); err != nil {
+			return nil, err
+		}
+		defer os.Remove(tempZip.Name())
+		zipFile = tempZip.Name()
+	}
+	r, err := os.Open(zipFile)
+	if err != nil {
+		return nil, fmt.Errorf("Could not open application package at %s: %w", ap.Path, err)
+	}
+	return r, nil
+}
+
 // Find an application package zip or directory below an application path
-func FindApplicationPackage(application string) (ApplicationPackage, error) {
+func ApplicationPackageFrom(application string) (ApplicationPackage, error) {
 	if isZip(application) {
 		return ApplicationPackage{Path: application}, nil
 	}
@@ -46,75 +106,119 @@ func FindApplicationPackage(application string) (ApplicationPackage, error) {
 	return ApplicationPackage{}, errors.New("Could not find an application package source in '" + application + "'")
 }
 
-func (ap *ApplicationPackage) IsZip() bool { return isZip(ap.Path) }
+func ApplicationFromString(s string) (ApplicationID, error) {
+	parts := strings.Split(s, ".")
+	if len(parts) != 3 {
+		return ApplicationID{}, fmt.Errorf("invalid application: %q", s)
+	}
+	return ApplicationID{Tenant: parts[0], Application: parts[1], Instance: parts[2]}, nil
+}
 
-func (ap *ApplicationPackage) HasCertificate() bool {
-	if ap.IsZip() {
-		return true // TODO: Consider looking inside zip to verify
+func ZoneFromString(s string) (ZoneID, error) {
+	parts := strings.Split(s, ".")
+	if len(parts) != 2 {
+		return ZoneID{}, fmt.Errorf("invalid zone: %q", s)
 	}
-	return util.PathExists(filepath.Join(ap.Path, "security", "clients.pem"))
+	return ZoneID{Environment: parts[0], Region: parts[1]}, nil
 }
 
-func isZip(filename string) bool { return filepath.Ext(filename) == ".zip" }
+func Prepare(deployment Deployment) (string, error) {
+	if deployment.IsCloud() {
+		return "", fmt.Errorf("%s: prepare is not supported", deployment)
+	}
+	// TODO: This doesn't work. A session ID must be explicitly created and then passed to the prepare call
+	// https://docs.vespa.ai/en/cloudconfig/deploy-rest-api-v2.html
+	u, err := url.Parse(deployment.TargetURL + "/application/v2/tenant/default/prepare")
+	if err != nil {
+		return "", err
+	}
+	return deploy(u, deployment.ApplicationSource)
+}
 
-func Deploy(prepare bool, application string, target string) (string, error) {
-	pkg, noSourceError := FindApplicationPackage(application)
-	if noSourceError != nil {
-		return "", noSourceError
+func Activate(deployment Deployment) (string, error) {
+	if deployment.IsCloud() {
+		return "", fmt.Errorf("%s: activate is not supported", deployment)
+	}
+	// TODO: This doesn't work. A session ID must be explicitly created and then passed to the activate call
+	// https://docs.vespa.ai/en/cloudconfig/deploy-rest-api-v2.html
+	u, err := url.Parse(deployment.TargetURL + "/application/v2/tenant/default/activate")
+	if err != nil {
+		return "", err
 	}
+	return deploy(u, deployment.ApplicationSource)
+}
 
-	zippedSource := pkg.Path
-	if !pkg.IsZip() { // create zip
-		tempZip, error := ioutil.TempFile("", "application.zip")
-		if error != nil {
-			return "", fmt.Errorf("Could not create a temporary zip file for the application package: %w", error)
+func Deploy(deployment Deployment) (string, error) {
+	path := "/application/v2/tenant/default/prepareandactivate"
+	if deployment.IsCloud() {
+		if deployment.APIKey == nil {
+			return "", fmt.Errorf("%s: missing api key", deployment.String())
 		}
-
-		error = zipDir(pkg.Path, tempZip.Name())
-		if error != nil {
-			return "", error
+		if deployment.KeyPair.Certificate == nil {
+			return "", fmt.Errorf("%s: missing certificate", deployment)
 		}
-		defer os.Remove(tempZip.Name())
-		zippedSource = tempZip.Name()
+		if deployment.KeyPair.PrivateKey == nil {
+			return "", fmt.Errorf("%s: missing private key", deployment)
+		}
+		if deployment.Zone.Environment == "" || deployment.Zone.Region == "" {
+			return "", fmt.Errorf("%s: missing zone", deployment)
+		}
+		path = fmt.Sprintf("/application/v4/tenant/%s/application/%s/instance/%s/deploy/%s-%s",
+			deployment.Application.Tenant,
+			deployment.Application.Application,
+			deployment.Application.Instance,
+			deployment.Zone.Environment,
+			deployment.Zone.Region)
+		return "", fmt.Errorf("cloud deployment is not implemented")
 	}
-
-	zipFileReader, zipFileError := os.Open(zippedSource)
-	if zipFileError != nil {
-		return "", fmt.Errorf("Could not open application package at %s: %w", pkg.Path, zipFileError)
+	u, err := url.Parse(deployment.TargetURL + path)
+	if err != nil {
+		return "", err
 	}
+	return deploy(u, deployment.ApplicationSource)
+}
 
-	var deployUrl *url.URL
-	if prepare {
-		deployUrl, _ = url.Parse(target + "/application/v2/tenant/default/prepare")
-	} else if application == "" {
-		deployUrl, _ = url.Parse(target + "/application/v2/tenant/default/activate")
-	} else {
-		deployUrl, _ = url.Parse(target + "/application/v2/tenant/default/prepareandactivate")
+func deploy(url *url.URL, applicationSource string) (string, error) {
+	pkg, err := ApplicationPackageFrom(applicationSource)
+	if err != nil {
+		return "", err
+	}
+	zipReader, err := pkg.zipReader()
+	if err != nil {
+		return "", err
 	}
+	if err := postApplicationPackage(url, zipReader); err != nil {
+		return "", err
+	}
+	return pkg.Path, nil
+}
 
+func postApplicationPackage(url *url.URL, zipReader io.Reader) error {
 	header := http.Header{}
 	header.Add("Content-Type", "application/zip")
 	request := &http.Request{
-		URL:    deployUrl,
+		URL:    url,
 		Method: "POST",
 		Header: header,
-		Body:   ioutil.NopCloser(zipFileReader),
+		Body:   io.NopCloser(zipReader),
 	}
 	serviceDescription := "Deploy service"
 	response, err := util.HttpDo(request, time.Minute*10, serviceDescription)
 	if err != nil {
-		return "", err
+		return err
 	}
 	defer response.Body.Close()
 
 	if response.StatusCode/100 == 4 {
-		return "", fmt.Errorf("Invalid application package (%s):\n%s", response.Status, util.ReaderToJSON(response.Body))
+		return fmt.Errorf("Invalid application package (%s):\n%s", response.Status, util.ReaderToJSON(response.Body))
 	} else if response.StatusCode != 200 {
-		return "", fmt.Errorf("Error from %s at %s (%s):\n%s", strings.ToLower(serviceDescription), request.URL.Host, response.Status, util.ReaderToJSON(response.Body))
+		return fmt.Errorf("Error from %s at %s (%s):\n%s", strings.ToLower(serviceDescription), request.URL.Host, response.Status, util.ReaderToJSON(response.Body))
 	}
-	return pkg.Path, nil
+	return nil
 }
 
+func isZip(filename string) bool { return filepath.Ext(filename) == ".zip" }
+
 func zipDir(dir string, destination string) error {
 	if filepath.IsAbs(dir) {
 		message := "Path must be relative, but '" + dir + "'"
diff --git a/client/go/vespa/deploy_test.go b/client/go/vespa/deploy_test.go
index c0995cbfe8c..fa2098b2231 100644
--- a/client/go/vespa/deploy_test.go
+++ b/client/go/vespa/deploy_test.go
@@ -8,7 +8,23 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFindApplicationPackage(t *testing.T) {
+func TestApplicationFromString(t *testing.T) {
+	app, err := ApplicationFromString("t1.a1.i1")
+	assert.Nil(t, err)
+	assert.Equal(t, ApplicationID{Tenant: "t1", Application: "a1", Instance: "i1"}, app)
+	_, err = ApplicationFromString("foo")
+	assert.NotNil(t, err)
+}
+
+func TestZoneFromString(t *testing.T) {
+	zone, err := ZoneFromString("dev.us-north-1")
+	assert.Nil(t, err)
+	assert.Equal(t, ZoneID{Environment: "dev", Region: "us-north-1"}, zone)
+	_, err = ZoneFromString("foo")
+	assert.NotNil(t, err)
+}
+
+func TestApplicationPackageFrom(t *testing.T) {
 	dir := t.TempDir()
 	var tests = []struct {
 		in   string
@@ -22,13 +38,13 @@ func TestFindApplicationPackage(t *testing.T) {
 
 	zipFile := filepath.Join(dir, "application.zip")
 	writeFile(t, zipFile)
-	pkg, err := FindApplicationPackage(zipFile)
+	pkg, err := ApplicationPackageFrom(zipFile)
 	assert.Nil(t, err)
 	assert.Equal(t, zipFile, pkg.Path)
 
 	for i, tt := range tests {
 		writeFile(t, tt.in)
-		pkg, err := FindApplicationPackage(dir)
+		pkg, err := ApplicationPackageFrom(dir)
 		if tt.fail {
 			assert.NotNil(t, err)
 		} else if pkg.Path != tt.out {