diff options
author | Jon Bratseth <bratseth@oath.com> | 2021-08-30 10:26:27 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-08-30 10:26:27 +0200 |
commit | 1a506104aec23f261142bb36fbf8b67eee75122f (patch) | |
tree | 03e330be06ea5f517fd9a24f42a36a84882b185b | |
parent | 357035c9b79f0498b586c1069cefb6303a75aa3e (diff) | |
parent | a71f4959d95c45c0c3282f7cf54b39c561ccf74f (diff) |
Merge pull request #18891 from vespa-engine/mpolden/vespa-cli-cloud
Refactor deploy command to support cloud
-rw-r--r-- | client/go/cmd/activate.go | 25 | ||||
-rw-r--r-- | client/go/cmd/cert.go | 27 | ||||
-rw-r--r-- | client/go/cmd/deploy.go | 97 | ||||
-rw-r--r-- | client/go/cmd/prepare.go | 25 | ||||
-rw-r--r-- | client/go/cmd/target.go | 15 | ||||
-rw-r--r-- | client/go/vespa/crypto.go | 17 | ||||
-rw-r--r-- | client/go/vespa/deploy.go | 182 | ||||
-rw-r--r-- | client/go/vespa/deploy_test.go | 22 |
8 files changed, 294 insertions, 116 deletions
diff --git a/client/go/cmd/activate.go b/client/go/cmd/activate.go deleted file mode 100644 index 70aed151a6d..00000000000 --- a/client/go/cmd/activate.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// vespa activate command -// Author: bratseth - -package cmd - -import ( - "github.com/spf13/cobra" -) - -func init() { - rootCmd.AddCommand(activateCmd) - addTargetFlag(activateCmd) -} - -// TODO: Implement and test - -var activateCmd = &cobra.Command{ - Use: "activate", - Short: "Activates (deploys) the previously prepared application package", - Long: `TODO`, - Run: func(cmd *cobra.Command, args []string) { - deploy(true, nil) - }, -} diff --git a/client/go/cmd/cert.go b/client/go/cmd/cert.go index 88d489fd229..f76a7041a82 100644 --- a/client/go/cmd/cert.go +++ b/client/go/cmd/cert.go @@ -15,16 +15,19 @@ import ( var overwriteCertificate bool func init() { - certCmd.Flags().BoolVarP(&overwriteCertificate, "force", "f", false, "Force overwrite of existing certificate and private key") rootCmd.AddCommand(certCmd) + certCmd.Flags().BoolVarP(&overwriteCertificate, "force", "f", false, "Force overwrite of existing certificate and private key") + certCmd.PersistentFlags().StringVarP(&applicationArg, applicationFlag, "a", "", "The application owning this certificate") + certCmd.MarkPersistentFlagRequired(applicationFlag) } var certCmd = &cobra.Command{ Use: "cert", Short: "Creates a new private key and self-signed certificate", - Long: `Applications in Vespa Cloud are required to secure their data plane with mutual TLS. ` + - `This command creates a self-signed certificate suitable for development purposes. ` + - `See https://cloud.vespa.ai/en/security-model for more information on the Vespa Cloud security model.`, + Long: "Applications in Vespa Cloud are required to secure their data plane with mutual TLS.\n\n" + + "This command creates a self-signed certificate suitable for development purposes.\n" + + "See https://cloud.vespa.ai/en/security-model for more information on the Vespa\n" + + "Cloud security model.", Run: func(cmd *cobra.Command, args []string) { var path string if len(args) > 0 { @@ -35,23 +38,28 @@ var certCmd = &cobra.Command{ fatalIfErr(err) } - pkg, err := vespa.FindApplicationPackage(path) + app, err := vespa.ApplicationFromString(applicationArg) + fatalIfErr(err) + + pkg, err := vespa.ApplicationPackageFrom(path) fatalIfErr(err) if pkg.HasCertificate() && !overwriteCertificate { log.Print("Certificate already exists. Use -f option to recreate") return } - // TODO: Consider writing key pair inside ~/.vespa/<app-name>/ instead so that vespa document commands can easily - // locate key pair + configDir, err := configDir(app.String()) + fatalIfErr(err) securityDir := filepath.Join(pkg.Path, "security") - privateKeyFile := filepath.Join(path, "data-plane-private-key.pem") - certificateFile := filepath.Join(path, "data-plane-public-cert.pem") pkgCertificateFile := filepath.Join(securityDir, "clients.pem") + privateKeyFile := filepath.Join(configDir, "data-plane-private-key.pem") + certificateFile := filepath.Join(configDir, "data-plane-public-cert.pem") keyPair, err := vespa.CreateKeyPair() fatalIfErr(err) + err = os.MkdirAll(configDir, 0755) + fatalIfErr(err) err = os.MkdirAll(securityDir, 0755) fatalIfErr(err) err = keyPair.WriteCertificateFile(pkgCertificateFile, overwriteCertificate) @@ -61,7 +69,6 @@ var certCmd = &cobra.Command{ err = keyPair.WritePrivateKeyFile(privateKeyFile, overwriteCertificate) fatalIfErr(err) - // TODO: Just use log package, which has Printf log.Printf("Certificate written to %s", color.Green(pkgCertificateFile)) log.Printf("Certificate written to %s", color.Green(certificateFile)) log.Printf("Private key written to %s", color.Green(privateKeyFile)) diff --git a/client/go/cmd/deploy.go b/client/go/cmd/deploy.go index ed7f7264fcc..76148f41291 100644 --- a/client/go/cmd/deploy.go +++ b/client/go/cmd/deploy.go @@ -6,14 +6,33 @@ package cmd import ( "log" + "os" + "path/filepath" "github.com/spf13/cobra" "github.com/vespa-engine/vespa/vespa" ) +const ( + zoneFlag = "zone" + applicationFlag = "application" +) + +var ( + zoneArg string + applicationArg string +) + func init() { rootCmd.AddCommand(deployCmd) + rootCmd.AddCommand(prepareCmd) + rootCmd.AddCommand(activateCmd) addTargetFlag(deployCmd) + addTargetFlag(prepareCmd) + addTargetFlag(activateCmd) + + deployCmd.PersistentFlags().StringVarP(&zoneArg, zoneFlag, "z", "dev.aws-us-east-1c", "The zone to use for deployment") + deployCmd.PersistentFlags().StringVarP(&applicationArg, applicationFlag, "a", "", "The application name to use for deployment") } var deployCmd = &cobra.Command{ @@ -21,19 +40,81 @@ var deployCmd = &cobra.Command{ Short: "Deploys (prepares and activates) an application package", Long: `TODO`, Run: func(cmd *cobra.Command, args []string) { - deploy(false, args) + d := vespa.Deployment{ + ApplicationSource: applicationSource(args), + TargetType: targetArg, + TargetURL: deployTarget(), + } + if d.IsCloud() { + var err error + d.Zone, err = vespa.ZoneFromString(zoneArg) + if err != nil { + log.Fatal(err) + } + d.Application, err = vespa.ApplicationFromString(applicationArg) + if err != nil { + log.Fatal(err) + } + + d.KeyPair, err = loadApplicationKeyPair(applicationArg) + if err != nil { + log.Fatal(err) + } + } + resolvedSrc, err := vespa.Deploy(d) + if err == nil { + log.Printf("Deployed %s successfully", color.Cyan(resolvedSrc)) + } else { + log.Print(err) + } + }, +} + +var prepareCmd = &cobra.Command{ + Use: "prepare", + Short: "Prepares an application package for activation", + Long: `TODO`, + Run: func(cmd *cobra.Command, args []string) { + resolvedSrc, err := vespa.Prepare(vespa.Deployment{ApplicationSource: applicationSource(args)}) + if err == nil { + log.Printf("Prepared %s successfully", color.Cyan(resolvedSrc)) + } else { + log.Print(color.Red(err)) + } + }, +} + +var activateCmd = &cobra.Command{ + Use: "activate", + Short: "Activates (deploys) the previously prepared application package", + Long: `TODO`, + Run: func(cmd *cobra.Command, args []string) { + resolvedSrc, err := vespa.Activate(vespa.Deployment{ApplicationSource: applicationSource(args)}) + if err == nil { + log.Printf("Activated %s successfully", color.Cyan(resolvedSrc)) + } else { + log.Print(color.Red(err)) + } }, } -func deploy(prepare bool, args []string) { - var application string +func loadApplicationKeyPair(application string) (vespa.PemKeyPair, error) { + configDir, err := configDir(application) + if err != nil { + return vespa.PemKeyPair{}, err + } + certificateFile := filepath.Join(configDir, "data-plane-public-cert.pem") + privateKeyFile := filepath.Join(configDir, "data-plane-private-key.pem") + return vespa.LoadKeyPair(privateKeyFile, certificateFile) +} + +func applicationSource(args []string) string { if len(args) > 0 { - application = args[0] + return args[0] } - path, err := vespa.Deploy(prepare, application, deployTarget()) + wd, err := os.Getwd() if err != nil { - log.Print(color.Red(err)) - } else { - log.Print("Deployed ", color.Green(path), " successfully") + log.Fatalf("Could not determine working directory: %s", err) } + return wd } diff --git a/client/go/cmd/prepare.go b/client/go/cmd/prepare.go deleted file mode 100644 index 5f31d0b8e23..00000000000 --- a/client/go/cmd/prepare.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// vespa prepare command -// Author: bratseth - -package cmd - -import ( - "github.com/spf13/cobra" -) - -func init() { - rootCmd.AddCommand(prepareCmd) - addTargetFlag(prepareCmd) -} - -// TODO: Implement and test - -var prepareCmd = &cobra.Command{ - Use: "prepare", - Short: "Prepares an application package for activation", - Long: `TODO`, - Run: func(cmd *cobra.Command, args []string) { - deploy(true, args) - }, -} diff --git a/client/go/cmd/target.go b/client/go/cmd/target.go index 79d3d6f32e8..d9acddc4dc9 100644 --- a/client/go/cmd/target.go +++ b/client/go/cmd/target.go @@ -11,9 +11,12 @@ import ( "github.com/spf13/cobra" ) -const flagName = "target" +const ( + targetFlag = "target" + cloudApi = "https://api.vespa-external.aws.oath.cloud" +) -var targetArgument string +var targetArg string type target struct { deploy string @@ -30,8 +33,8 @@ const ( ) func addTargetFlag(command *cobra.Command) { - command.PersistentFlags().StringVarP(&targetArgument, flagName, "t", "local", "The name or URL of the recipient of this command") - bindFlagToConfig(flagName, command) + command.PersistentFlags().StringVarP(&targetArg, targetFlag, "t", "local", "The name or URL of the recipient of this command") + bindFlagToConfig(targetFlag, command) } func deployTarget() string { @@ -47,7 +50,7 @@ func documentTarget() string { } func getTarget(targetContext context) *target { - targetValue, err := getOption(flagName) + targetValue, err := getOption(targetFlag) if err != nil { log.Fatalf("a valid target must be specified") } @@ -80,7 +83,7 @@ func getTarget(targetContext context) *target { } if targetValue == "cloud" { - panic("cloud target is not implemented") + return &target{deploy: cloudApi} } log.Printf("Unknown target '%s': Use %s, %s or an URL", color.Red(targetValue), color.Cyan("local"), color.Cyan("cloud")) diff --git a/client/go/vespa/crypto.go b/client/go/vespa/crypto.go index b9e41d81576..a410f65c620 100644 --- a/client/go/vespa/crypto.go +++ b/client/go/vespa/crypto.go @@ -83,6 +83,23 @@ func CreateKeyPair() (PemKeyPair, error) { return PemKeyPair{Certificate: pemCertificate, PrivateKey: pemPrivateKey}, nil } +// LoadKeyPair reads a key pair located in privateKeyFile and certificateFile. +func LoadKeyPair(privateKeyFile, certificateFile string) (PemKeyPair, error) { + var ( + kp PemKeyPair + err error + ) + kp.PrivateKey, err = os.ReadFile(privateKeyFile) + if err != nil { + return PemKeyPair{}, err + } + kp.Certificate, err = os.ReadFile(certificateFile) + if err != nil { + return PemKeyPair{}, err + } + return kp, err +} + func randomSerialNumber() (*big.Int, error) { serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) return rand.Int(rand.Reader, serialNumberLimit) diff --git a/client/go/vespa/deploy.go b/client/go/vespa/deploy.go index 9973d3fd490..037c92b5458 100644 --- a/client/go/vespa/deploy.go +++ b/client/go/vespa/deploy.go @@ -20,12 +20,72 @@ import ( "github.com/vespa-engine/vespa/util" ) +type ApplicationID struct { + Tenant string + Application string + Instance string +} + +type ZoneID struct { + Environment string + Region string +} + +type Deployment struct { + ApplicationSource string + TargetType string + TargetURL string + Application ApplicationID + Zone ZoneID + KeyPair PemKeyPair + APIKey []byte +} + type ApplicationPackage struct { Path string } +func (a ApplicationID) String() string { + return fmt.Sprintf("%s.%s.%s", a.Tenant, a.Application, a.Instance) +} + +func (d Deployment) String() string { + return fmt.Sprintf("deployment of %s to %s target", d.Application, d.TargetType) +} + +func (d *Deployment) IsCloud() bool { return d.TargetType == "cloud" } + +func (ap *ApplicationPackage) IsZip() bool { return isZip(ap.Path) } + +func (ap *ApplicationPackage) HasCertificate() bool { + if ap.IsZip() { + return true // TODO: Consider looking inside zip to verify + } + return util.PathExists(filepath.Join(ap.Path, "security", "clients.pem")) +} + +func (ap *ApplicationPackage) zipReader() (io.ReadCloser, error) { + zipFile := ap.Path + if !ap.IsZip() { + tempZip, error := ioutil.TempFile("", "application.zip") + if error != nil { + return nil, fmt.Errorf("Could not create a temporary zip file for the application package: %w", error) + } + if err := zipDir(ap.Path, tempZip.Name()); err != nil { + return nil, err + } + defer os.Remove(tempZip.Name()) + zipFile = tempZip.Name() + } + r, err := os.Open(zipFile) + if err != nil { + return nil, fmt.Errorf("Could not open application package at %s: %w", ap.Path, err) + } + return r, nil +} + // Find an application package zip or directory below an application path -func FindApplicationPackage(application string) (ApplicationPackage, error) { +func ApplicationPackageFrom(application string) (ApplicationPackage, error) { if isZip(application) { return ApplicationPackage{Path: application}, nil } @@ -46,75 +106,119 @@ func FindApplicationPackage(application string) (ApplicationPackage, error) { return ApplicationPackage{}, errors.New("Could not find an application package source in '" + application + "'") } -func (ap *ApplicationPackage) IsZip() bool { return isZip(ap.Path) } +func ApplicationFromString(s string) (ApplicationID, error) { + parts := strings.Split(s, ".") + if len(parts) != 3 { + return ApplicationID{}, fmt.Errorf("invalid application: %q", s) + } + return ApplicationID{Tenant: parts[0], Application: parts[1], Instance: parts[2]}, nil +} -func (ap *ApplicationPackage) HasCertificate() bool { - if ap.IsZip() { - return true // TODO: Consider looking inside zip to verify +func ZoneFromString(s string) (ZoneID, error) { + parts := strings.Split(s, ".") + if len(parts) != 2 { + return ZoneID{}, fmt.Errorf("invalid zone: %q", s) } - return util.PathExists(filepath.Join(ap.Path, "security", "clients.pem")) + return ZoneID{Environment: parts[0], Region: parts[1]}, nil } -func isZip(filename string) bool { return filepath.Ext(filename) == ".zip" } +func Prepare(deployment Deployment) (string, error) { + if deployment.IsCloud() { + return "", fmt.Errorf("%s: prepare is not supported", deployment) + } + // TODO: This doesn't work. A session ID must be explicitly created and then passed to the prepare call + // https://docs.vespa.ai/en/cloudconfig/deploy-rest-api-v2.html + u, err := url.Parse(deployment.TargetURL + "/application/v2/tenant/default/prepare") + if err != nil { + return "", err + } + return deploy(u, deployment.ApplicationSource) +} -func Deploy(prepare bool, application string, target string) (string, error) { - pkg, noSourceError := FindApplicationPackage(application) - if noSourceError != nil { - return "", noSourceError +func Activate(deployment Deployment) (string, error) { + if deployment.IsCloud() { + return "", fmt.Errorf("%s: activate is not supported", deployment) + } + // TODO: This doesn't work. A session ID must be explicitly created and then passed to the activate call + // https://docs.vespa.ai/en/cloudconfig/deploy-rest-api-v2.html + u, err := url.Parse(deployment.TargetURL + "/application/v2/tenant/default/activate") + if err != nil { + return "", err } + return deploy(u, deployment.ApplicationSource) +} - zippedSource := pkg.Path - if !pkg.IsZip() { // create zip - tempZip, error := ioutil.TempFile("", "application.zip") - if error != nil { - return "", fmt.Errorf("Could not create a temporary zip file for the application package: %w", error) +func Deploy(deployment Deployment) (string, error) { + path := "/application/v2/tenant/default/prepareandactivate" + if deployment.IsCloud() { + if deployment.APIKey == nil { + return "", fmt.Errorf("%s: missing api key", deployment.String()) } - - error = zipDir(pkg.Path, tempZip.Name()) - if error != nil { - return "", error + if deployment.KeyPair.Certificate == nil { + return "", fmt.Errorf("%s: missing certificate", deployment) } - defer os.Remove(tempZip.Name()) - zippedSource = tempZip.Name() + if deployment.KeyPair.PrivateKey == nil { + return "", fmt.Errorf("%s: missing private key", deployment) + } + if deployment.Zone.Environment == "" || deployment.Zone.Region == "" { + return "", fmt.Errorf("%s: missing zone", deployment) + } + path = fmt.Sprintf("/application/v4/tenant/%s/application/%s/instance/%s/deploy/%s-%s", + deployment.Application.Tenant, + deployment.Application.Application, + deployment.Application.Instance, + deployment.Zone.Environment, + deployment.Zone.Region) + return "", fmt.Errorf("cloud deployment is not implemented") } - - zipFileReader, zipFileError := os.Open(zippedSource) - if zipFileError != nil { - return "", fmt.Errorf("Could not open application package at %s: %w", pkg.Path, zipFileError) + u, err := url.Parse(deployment.TargetURL + path) + if err != nil { + return "", err } + return deploy(u, deployment.ApplicationSource) +} - var deployUrl *url.URL - if prepare { - deployUrl, _ = url.Parse(target + "/application/v2/tenant/default/prepare") - } else if application == "" { - deployUrl, _ = url.Parse(target + "/application/v2/tenant/default/activate") - } else { - deployUrl, _ = url.Parse(target + "/application/v2/tenant/default/prepareandactivate") +func deploy(url *url.URL, applicationSource string) (string, error) { + pkg, err := ApplicationPackageFrom(applicationSource) + if err != nil { + return "", err + } + zipReader, err := pkg.zipReader() + if err != nil { + return "", err } + if err := postApplicationPackage(url, zipReader); err != nil { + return "", err + } + return pkg.Path, nil +} +func postApplicationPackage(url *url.URL, zipReader io.Reader) error { header := http.Header{} header.Add("Content-Type", "application/zip") request := &http.Request{ - URL: deployUrl, + URL: url, Method: "POST", Header: header, - Body: ioutil.NopCloser(zipFileReader), + Body: io.NopCloser(zipReader), } serviceDescription := "Deploy service" response, err := util.HttpDo(request, time.Minute*10, serviceDescription) if err != nil { - return "", err + return err } defer response.Body.Close() if response.StatusCode/100 == 4 { - return "", fmt.Errorf("Invalid application package (%s):\n%s", response.Status, util.ReaderToJSON(response.Body)) + return fmt.Errorf("Invalid application package (%s):\n%s", response.Status, util.ReaderToJSON(response.Body)) } else if response.StatusCode != 200 { - return "", fmt.Errorf("Error from %s at %s (%s):\n%s", strings.ToLower(serviceDescription), request.URL.Host, response.Status, util.ReaderToJSON(response.Body)) + return fmt.Errorf("Error from %s at %s (%s):\n%s", strings.ToLower(serviceDescription), request.URL.Host, response.Status, util.ReaderToJSON(response.Body)) } - return pkg.Path, nil + return nil } +func isZip(filename string) bool { return filepath.Ext(filename) == ".zip" } + func zipDir(dir string, destination string) error { if filepath.IsAbs(dir) { message := "Path must be relative, but '" + dir + "'" diff --git a/client/go/vespa/deploy_test.go b/client/go/vespa/deploy_test.go index c0995cbfe8c..fa2098b2231 100644 --- a/client/go/vespa/deploy_test.go +++ b/client/go/vespa/deploy_test.go @@ -8,7 +8,23 @@ import ( "github.com/stretchr/testify/assert" ) -func TestFindApplicationPackage(t *testing.T) { +func TestApplicationFromString(t *testing.T) { + app, err := ApplicationFromString("t1.a1.i1") + assert.Nil(t, err) + assert.Equal(t, ApplicationID{Tenant: "t1", Application: "a1", Instance: "i1"}, app) + _, err = ApplicationFromString("foo") + assert.NotNil(t, err) +} + +func TestZoneFromString(t *testing.T) { + zone, err := ZoneFromString("dev.us-north-1") + assert.Nil(t, err) + assert.Equal(t, ZoneID{Environment: "dev", Region: "us-north-1"}, zone) + _, err = ZoneFromString("foo") + assert.NotNil(t, err) +} + +func TestApplicationPackageFrom(t *testing.T) { dir := t.TempDir() var tests = []struct { in string @@ -22,13 +38,13 @@ func TestFindApplicationPackage(t *testing.T) { zipFile := filepath.Join(dir, "application.zip") writeFile(t, zipFile) - pkg, err := FindApplicationPackage(zipFile) + pkg, err := ApplicationPackageFrom(zipFile) assert.Nil(t, err) assert.Equal(t, zipFile, pkg.Path) for i, tt := range tests { writeFile(t, tt.in) - pkg, err := FindApplicationPackage(dir) + pkg, err := ApplicationPackageFrom(dir) if tt.fail { assert.NotNil(t, err) } else if pkg.Path != tt.out { |