diff options
-rw-r--r-- | client/go/cmd/command_tester.go | 2 | ||||
-rw-r--r-- | client/go/cmd/deploy.go | 15 | ||||
-rw-r--r-- | client/go/cmd/helpers.go | 31 | ||||
-rw-r--r-- | client/go/cmd/prod.go | 76 | ||||
-rw-r--r-- | client/go/cmd/prod_test.go | 48 | ||||
-rw-r--r-- | client/go/cmd/query_test.go | 2 | ||||
-rw-r--r-- | client/go/cmd/testdata/applications/withDeployment/target/application-test.zip | bin | 0 -> 23061 bytes | |||
-rw-r--r-- | client/go/cmd/testdata/applications/withDeployment/target/application.zip | bin | 0 -> 11942 bytes | |||
-rw-r--r-- | client/go/vespa/deploy.go | 147 | ||||
-rw-r--r-- | client/go/vespa/xml/config.go | 4 |
10 files changed, 284 insertions, 41 deletions
diff --git a/client/go/cmd/command_tester.go b/client/go/cmd/command_tester.go index 201adb8cab7..2d2de6a201c 100644 --- a/client/go/cmd/command_tester.go +++ b/client/go/cmd/command_tester.go @@ -58,6 +58,8 @@ func execute(cmd command, t *testing.T, client *mockHttpClient) (string, string) os.Setenv("VESPA_CLI_CACHE_DIR", cmd.cacheDir) // Reset flags to their default value - persistent flags in Cobra persists over tests + // TODO: Due to the bad design of viper, the only proper fix is to get rid of global state by moving each command to + // their own sub-package rootCmd.Flags().VisitAll(resetFlag) documentCmd.Flags().VisitAll(resetFlag) diff --git a/client/go/cmd/deploy.go b/client/go/cmd/deploy.go index 7c9c3d8cc02..034dac2e67b 100644 --- a/client/go/cmd/deploy.go +++ b/client/go/cmd/deploy.go @@ -62,20 +62,7 @@ $ vespa deploy -t cloud -z perf.aws-us-east-1c`, return } target := getTarget() - opts := vespa.DeploymentOpts{ApplicationPackage: pkg, Target: target} - if opts.IsCloud() { - deployment := deploymentFromArgs() - if !opts.ApplicationPackage.HasCertificate() { - fatalErrHint(fmt.Errorf("Missing certificate in application package"), "Applications in Vespa Cloud require a certificate", "Try 'vespa cert'") - return - } - opts.APIKey, err = cfg.ReadAPIKey(deployment.Application.Tenant) - if err != nil { - fatalErrHint(err, "Deployment to cloud requires an API key. Try 'vespa api-key'") - return - } - opts.Deployment = deployment - } + opts := getDeploymentOpts(cfg, pkg, target) if sessionOrRunID, err := vespa.Deploy(opts); err == nil { if opts.IsCloud() { printSuccess("Triggered deployment of ", color.Cyan(pkg.Path), " with run ID ", color.Cyan(sessionOrRunID)) diff --git a/client/go/cmd/helpers.go b/client/go/cmd/helpers.go index b5525cf11fe..4768290e33e 100644 --- a/client/go/cmd/helpers.go +++ b/client/go/cmd/helpers.go @@ -30,7 +30,9 @@ func fatalErr(err error, msg ...interface{}) { } func printErrHint(err error, hints ...string) { - printErr(nil, err.Error()) + if err != nil { + printErr(nil, err.Error()) + } for _, hint := range hints { fmt.Fprintln(stderr, color.Cyan("Hint:"), hint) } @@ -140,9 +142,10 @@ func getService(service string, sessionOrRunID int64) *vespa.Service { return s } +func getSystem() string { return os.Getenv("VESPA_CLI_CLOUD_SYSTEM") } + func getConsoleURL() string { - system := os.Getenv("VESPA_CLI_CLOUD_SYSTEM") - if system == "publiccd" { + if getSystem() == "publiccd" { return "https://console-cd.vespa.oath.cloud" } return "https://console.vespa.oath.cloud" @@ -150,8 +153,7 @@ func getConsoleURL() string { } func getApiURL() string { - system := os.Getenv("VESPA_CLI_CLOUD_SYSTEM") - if system == "publiccd" { + if getSystem() == "publiccd" { return "https://api.vespa-external-cd.aws.oath.cloud:4443" } return "https://api.vespa-external.aws.oath.cloud:4443" @@ -221,3 +223,22 @@ func waitForService(service string, sessionOrRunID int64) { fatalErr(err, s.Description(), " at ", color.Cyan(s.BaseURL), " is ", color.Red("not ready")) } } + +func getDeploymentOpts(cfg *Config, pkg vespa.ApplicationPackage, target vespa.Target) vespa.DeploymentOpts { + opts := vespa.DeploymentOpts{ApplicationPackage: pkg, Target: target} + if opts.IsCloud() { + deployment := deploymentFromArgs() + if !opts.ApplicationPackage.HasCertificate() { + fatalErrHint(fmt.Errorf("Missing certificate in application package"), "Applications in Vespa Cloud require a certificate", "Try 'vespa cert'") + return opts + } + var err error + opts.APIKey, err = cfg.ReadAPIKey(deployment.Application.Tenant) + if err != nil { + fatalErrHint(err, "Deployment to cloud requires an API key. Try 'vespa api-key'") + return opts + } + opts.Deployment = deployment + } + return opts +} diff --git a/client/go/cmd/prod.go b/client/go/cmd/prod.go index 15d9edc18a7..382ede0fae8 100644 --- a/client/go/cmd/prod.go +++ b/client/go/cmd/prod.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io/ioutil" + "log" "os" "path/filepath" "strings" @@ -20,6 +21,7 @@ import ( func init() { rootCmd.AddCommand(prodCmd) prodCmd.AddCommand(prodInitCmd) + prodCmd.AddCommand(prodSubmitCmd) } var prodCmd = &cobra.Command{ @@ -60,7 +62,7 @@ https://cloud.vespa.ai/en/reference/deployment`, } if pkg.IsZip() { fatalErrHint(fmt.Errorf("Cannot modify compressed application package %s", pkg.Path), - "Try running 'mvn clean' and re-running this command") + "Try running 'mvn clean' and run this command again") return } @@ -95,6 +97,69 @@ https://cloud.vespa.ai/en/reference/deployment`, }, } +var prodSubmitCmd = &cobra.Command{ + Use: "submit", + Short: "Submit your application for production deployment", + Long: `Submit your application for production deployment. + +This commands uploads your application package to Vespa Cloud and deploys it to +the production zones specified in deployment.xml. + +Nodes are allocated to your application according to resources specified in +services.xml. + +While submitting an application from a local development environment is +supported, it's strongly recommended that production deployments are performed +by a continuous build system. + +For more information about production deployments in Vespa Cloud see: +https://cloud.vespa.ai/en/getting-to-production +https://cloud.vespa.ai/en/automated-deployments`, + DisableAutoGenTag: true, + Example: `$ mvn package +$ vespa prod submit`, + Run: func(cmd *cobra.Command, args []string) { + target := getTarget() + if target.Type() != "cloud" { + fatalErr(fmt.Errorf("%s target cannot deploy to Vespa Cloud", target.Type())) + return + } + appSource := applicationSource(args) + pkg, err := vespa.FindApplicationPackage(appSource, true) + if err != nil { + fatalErr(err) + return + } + cfg, err := LoadConfig() + if err != nil { + fatalErr(err, "Could not load config") + return + } + if !pkg.HasDeployment() { + fatalErrHint(fmt.Errorf("No deployment.xml found"), "Try creating one with vespa prod init") + return + } + if !pkg.IsJava() { + // TODO: Loosen this requirement when we start supporting applications with Java in production + fatalErrHint(fmt.Errorf("No jar files found in %s", pkg.Path), "Only applications containing Java components are currently supported") + return + } + isCI := os.Getenv("CI") != "" + if !isCI { + fmt.Fprintln(stderr, color.Yellow("Warning:"), "Submitting from a non-CI environment is discouraged") + printErrHint(nil, "See https://cloud.vespa.ai/en/getting-to-production for best practices") + } + opts := getDeploymentOpts(cfg, pkg, target) + if err := vespa.Submit(opts); err != nil { + fatalErr(err, "Could not submit application for deployment") + } else { + printSuccess("Submitted ", color.Cyan(pkg.Path), " for deployment") + log.Printf("See %s for deployment progress\n", color.Cyan(fmt.Sprintf("%s/tenant/%s/application/%s/prod/deployment", + getConsoleURL(), opts.Deployment.Application.Tenant, opts.Deployment.Application.Application))) + } + }, +} + func writeWithBackup(pkg vespa.ApplicationPackage, filename, contents string) error { dst := filepath.Join(pkg.Path, filename) if util.PathExists(dst) { @@ -133,6 +198,13 @@ func updateRegions(r *bufio.Reader, deploymentXML xml.Deployment) xml.Deployment if err := deploymentXML.Replace("prod", "region", regionElements); err != nil { fatalErr(err, "Could not update region elements in deployment.xml") } + // TODO: Some sample apps come with production <test> elements, but not necessarily working production tests, we + // therefore remove <test> elements here. + // This can be improved by supporting <test> elements in xml package and allow specifying testing as part of + // region prompt, e.g. region1;test,region2 + if err := deploymentXML.Replace("prod", "test", nil); err != nil { + fatalErr(err, "Could not remove test elements in deployment.xml") + } return deploymentXML } @@ -152,7 +224,7 @@ func promptRegions(r *bufio.Reader, deploymentXML xml.Deployment) string { validator := func(input string) error { regions := strings.Split(input, ",") for _, r := range regions { - if !xml.ValidProdRegion(r) { + if !xml.IsProdRegion(r, getSystem()) { return fmt.Errorf("invalid region %s", r) } } diff --git a/client/go/cmd/prod_test.go b/client/go/cmd/prod_test.go index 5abb6d24577..4ce6112122a 100644 --- a/client/go/cmd/prod_test.go +++ b/client/go/cmd/prod_test.go @@ -2,6 +2,7 @@ package cmd import ( "bytes" + "io" "io/ioutil" "os" "path/filepath" @@ -82,6 +83,7 @@ func readFileString(t *testing.T, filename string) string { func createApplication(t *testing.T, pkgDir string) { appDir := filepath.Join(pkgDir, "src", "main", "application") + targetDir := filepath.Join(pkgDir, "target") if err := os.MkdirAll(appDir, 0755); err != nil { t.Fatal(err) } @@ -115,4 +117,50 @@ func createApplication(t *testing.T, pkgDir string) { if err := ioutil.WriteFile(filepath.Join(appDir, "services.xml"), []byte(servicesXML), 0644); err != nil { t.Fatal(err) } + if err := os.MkdirAll(targetDir, 0755); err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(filepath.Join(pkgDir, "pom.xml"), []byte(""), 0644); err != nil { + t.Fatal(err) + } +} + +func TestProdSubmit(t *testing.T) { + homeDir := filepath.Join(t.TempDir(), ".vespa") + pkgDir := filepath.Join(t.TempDir(), "app") + createApplication(t, pkgDir) + + httpClient := &mockHttpClient{} + httpClient.NextResponse(200, `ok`) + execute(command{homeDir: homeDir, args: []string{"config", "set", "application", "t1.a1.i1"}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "cloud"}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"api-key"}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"cert", pkgDir}}, t, httpClient) + + // Copy an application package pre-assambled with mvn package + testAppDir := filepath.Join("testdata", "applications", "withDeployment", "target") + zipFile := filepath.Join(testAppDir, "application.zip") + copyFile(t, filepath.Join(pkgDir, "target", "application.zip"), zipFile) + testZipFile := filepath.Join(testAppDir, "application-test.zip") + copyFile(t, filepath.Join(pkgDir, "target", "application-test.zip"), testZipFile) + + out, _ := execute(command{homeDir: homeDir, args: []string{"prod", "submit", pkgDir}}, t, httpClient) + assert.Contains(t, out, "Success: Submitted") + assert.Contains(t, out, "See https://console.vespa.oath.cloud/tenant/t1/application/a1/prod/deployment for deployment progress") +} + +func copyFile(t *testing.T, dstFilename, srcFilename string) { + dst, err := os.Create(dstFilename) + if err != nil { + t.Fatal(err) + } + defer dst.Close() + src, err := os.Open(srcFilename) + if err != nil { + t.Fatal(err) + } + defer src.Close() + if _, err := io.Copy(dst, src); err != nil { + t.Fatal(err) + } } diff --git a/client/go/cmd/query_test.go b/client/go/cmd/query_test.go index 81dc03766be..ec6c3063906 100644 --- a/client/go/cmd/query_test.go +++ b/client/go/cmd/query_test.go @@ -45,12 +45,12 @@ func TestServerError(t *testing.T) { func assertQuery(t *testing.T, expectedQuery string, query ...string) { client := &mockHttpClient{} - queryURL := queryServiceURL(client) client.NextResponse(200, "{\"query\":\"result\"}") assert.Equal(t, "{\n \"query\": \"result\"\n}\n", executeCommand(t, client, []string{"query"}, query), "query output") + queryURL := queryServiceURL(client) assert.Equal(t, queryURL+"/search/"+expectedQuery, client.lastRequest.URL.String()) } diff --git a/client/go/cmd/testdata/applications/withDeployment/target/application-test.zip b/client/go/cmd/testdata/applications/withDeployment/target/application-test.zip Binary files differnew file mode 100644 index 00000000000..8a1707b9cee --- /dev/null +++ b/client/go/cmd/testdata/applications/withDeployment/target/application-test.zip diff --git a/client/go/cmd/testdata/applications/withDeployment/target/application.zip b/client/go/cmd/testdata/applications/withDeployment/target/application.zip Binary files differnew file mode 100644 index 00000000000..da23c2ff437 --- /dev/null +++ b/client/go/cmd/testdata/applications/withDeployment/target/application.zip diff --git a/client/go/vespa/deploy.go b/client/go/vespa/deploy.go index 908b3772b70..3718c7d813a 100644 --- a/client/go/vespa/deploy.go +++ b/client/go/vespa/deploy.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "io/ioutil" + "mime/multipart" "net/http" "net/url" "os" @@ -49,7 +50,8 @@ type DeploymentOpts struct { } type ApplicationPackage struct { - Path string + Path string + TestPath string } func (a ApplicationID) String() string { @@ -79,6 +81,15 @@ func (d *DeploymentOpts) url(path string) (*url.URL, error) { } func (ap *ApplicationPackage) HasCertificate() bool { + return ap.hasFile(filepath.Join("security", "clients.pem"), "security/clients.pem") +} + +func (ap *ApplicationPackage) HasDeployment() bool { return ap.hasFile("deployment.xml", "") } + +func (ap *ApplicationPackage) hasFile(filename, zipName string) bool { + if zipName == "" { + zipName = filename + } if ap.IsZip() { r, err := zip.OpenReader(ap.Path) if err != nil { @@ -86,35 +97,58 @@ func (ap *ApplicationPackage) HasCertificate() bool { } defer r.Close() for _, f := range r.File { - if f.Name == "security/clients.pem" { + if f.Name == zipName { return true } } return false } - return util.PathExists(filepath.Join(ap.Path, "security", "clients.pem")) + return util.PathExists(filepath.Join(ap.Path, filename)) } func (ap *ApplicationPackage) IsZip() bool { return isZip(ap.Path) } -func (ap *ApplicationPackage) zipReader() (io.ReadCloser, error) { +func (ap *ApplicationPackage) IsJava() bool { + if ap.IsZip() { + r, err := zip.OpenReader(ap.Path) + if err != nil { + return false + } + defer r.Close() + for _, f := range r.File { + if filepath.Ext(f.Name) == ".jar" { + return true + } + } + return false + } + return util.PathExists(filepath.Join(ap.Path, "pom.xml")) +} + +func (ap *ApplicationPackage) zipReader(test bool) (io.ReadCloser, error) { zipFile := ap.Path + if test { + zipFile = ap.TestPath + } if !ap.IsZip() { - tempZip, error := ioutil.TempFile("", "application.zip") - if error != nil { - return nil, fmt.Errorf("Could not create a temporary zip file for the application package: %w", error) + tempZip, err := ioutil.TempFile("", "vespa") + if err != nil { + return nil, fmt.Errorf("Could not create a temporary zip file for the application package: %w", err) } + defer func() { + tempZip.Close() + os.Remove(tempZip.Name()) + }() if err := zipDir(ap.Path, tempZip.Name()); err != nil { return nil, err } - defer os.Remove(tempZip.Name()) zipFile = tempZip.Name() } - r, err := os.Open(zipFile) + f, err := os.Open(zipFile) if err != nil { return nil, fmt.Errorf("Could not open application package at %s: %w", ap.Path, err) } - return r, nil + return f, nil } // FindApplicationPackage finds the path to an application package from the zip file or directory zipOrDir. @@ -125,7 +159,8 @@ func FindApplicationPackage(zipOrDir string, requirePackaging bool) (Application if util.PathExists(filepath.Join(zipOrDir, "pom.xml")) { zip := filepath.Join(zipOrDir, "target", "application.zip") if util.PathExists(zip) { - return ApplicationPackage{Path: zip}, nil + testZip := filepath.Join(zipOrDir, "target", "application-test.zip") + return ApplicationPackage{Path: zip, TestPath: testZip}, nil } if requirePackaging { return ApplicationPackage{}, errors.New("pom.xml exists but no target/application.zip. Run mvn package first") @@ -214,11 +249,8 @@ func Activate(sessionID int64, deployment DeploymentOpts) error { func Deploy(opts DeploymentOpts) (int64, error) { path := "/application/v2/tenant/default/prepareandactivate" if opts.IsCloud() { - if !opts.ApplicationPackage.HasCertificate() { - return 0, fmt.Errorf("%s: missing certificate in package", opts) - } - if opts.APIKey == nil { - return 0, fmt.Errorf("%s: missing api key", opts.String()) + if err := checkDeploymentOpts(opts); err != nil { + return 0, err } if opts.Deployment.Zone.Environment == "" || opts.Deployment.Zone.Region == "" { return 0, fmt.Errorf("%s: missing zone", opts) @@ -237,8 +269,89 @@ func Deploy(opts DeploymentOpts) (int64, error) { return uploadApplicationPackage(u, opts) } +func copyToPart(dst *multipart.Writer, src io.Reader, fieldname, filename string) error { + var part io.Writer + var err error + if filename == "" { + part, err = dst.CreateFormField(fieldname) + } else { + part, err = dst.CreateFormFile(fieldname, filename) + } + if err != nil { + return err + } + if _, err := io.Copy(part, src); err != nil { + return err + } + return nil +} + +func Submit(opts DeploymentOpts) error { + if !opts.IsCloud() { + return fmt.Errorf("%s: submit is unsupported", opts) + } + if err := checkDeploymentOpts(opts); err != nil { + return err + } + path := fmt.Sprintf("/application/v4/tenant/%s/application/%s/submit", opts.Deployment.Application.Tenant, opts.Deployment.Application.Application) + u, err := opts.url(path) + if err != nil { + return err + } + var body bytes.Buffer + writer := multipart.NewWriter(&body) + if err := copyToPart(writer, strings.NewReader("{}"), "submitOptions", ""); err != nil { + return err + } + applicationZip, err := opts.ApplicationPackage.zipReader(false) + if err != nil { + return err + } + if err := copyToPart(writer, applicationZip, "applicationZip", "application.zip"); err != nil { + return err + } + testApplicationZip, err := opts.ApplicationPackage.zipReader(true) + if err != nil { + return err + } + if err := copyToPart(writer, testApplicationZip, "applicationTestZip", "application-test.zip"); err != nil { + return err + } + if err := writer.Close(); err != nil { + return err + } + request := &http.Request{ + URL: u, + Method: "POST", + Body: ioutil.NopCloser(&body), + Header: make(http.Header), + } + request.Header.Set("Content-Type", writer.FormDataContentType()) + signer := NewRequestSigner(opts.Deployment.Application.SerializedForm(), opts.APIKey) + if err := signer.SignRequest(request); err != nil { + return err + } + serviceDescription := "Submit service" + response, err := util.HttpDo(request, time.Minute*10, serviceDescription) + if err != nil { + return err + } + defer response.Body.Close() + return checkResponse(request, response, serviceDescription) +} + +func checkDeploymentOpts(opts DeploymentOpts) error { + if !opts.ApplicationPackage.HasCertificate() { + return fmt.Errorf("%s: missing certificate in package", opts) + } + if opts.APIKey == nil { + return fmt.Errorf("%s: missing api key", opts.String()) + } + return nil +} + func uploadApplicationPackage(url *url.URL, opts DeploymentOpts) (int64, error) { - zipReader, err := opts.ApplicationPackage.zipReader() + zipReader, err := opts.ApplicationPackage.zipReader(false) if err != nil { return 0, err } diff --git a/client/go/vespa/xml/config.go b/client/go/vespa/xml/config.go index a30db238afb..e900b50cbb0 100644 --- a/client/go/vespa/xml/config.go +++ b/client/go/vespa/xml/config.go @@ -193,8 +193,8 @@ func ParseNodeCount(s string) (int, int, error) { return 0, 0, parseErr } -// ValidProdRegion returns whether string s is a valid production region. -func ValidProdRegion(s string, system string) bool { +// IsProdRegion returns whether string s is a valid production region. +func IsProdRegion(s string, system string) bool { if system == "publiccd" { return s == "aws-us-east-1c" } |