diff options
author | Martin Polden <mpolden@mpolden.no> | 2021-10-06 14:36:27 +0200 |
---|---|---|
committer | Martin Polden <mpolden@mpolden.no> | 2021-10-07 09:33:53 +0200 |
commit | 27df9f15770e537a65466ef1a753dc60b8ec6163 (patch) | |
tree | 61fc60e6c0699ba0ca46885ce03e76be3b3537c4 /client/go/cmd | |
parent | a3530f08a4611542f5a318563d96fec63dbf0bb9 (diff) |
Implement vespa prod submit
Diffstat (limited to 'client/go/cmd')
-rw-r--r-- | client/go/cmd/command_tester.go | 2 | ||||
-rw-r--r-- | client/go/cmd/deploy.go | 15 | ||||
-rw-r--r-- | client/go/cmd/helpers.go | 31 | ||||
-rw-r--r-- | client/go/cmd/prod.go | 76 | ||||
-rw-r--r-- | client/go/cmd/prod_test.go | 48 | ||||
-rw-r--r-- | client/go/cmd/query_test.go | 2 | ||||
-rw-r--r-- | client/go/cmd/testdata/applications/withDeployment/target/application-test.zip | bin | 0 -> 23061 bytes | |||
-rw-r--r-- | client/go/cmd/testdata/applications/withDeployment/target/application.zip | bin | 0 -> 11942 bytes |
8 files changed, 152 insertions, 22 deletions
diff --git a/client/go/cmd/command_tester.go b/client/go/cmd/command_tester.go index 201adb8cab7..2d2de6a201c 100644 --- a/client/go/cmd/command_tester.go +++ b/client/go/cmd/command_tester.go @@ -58,6 +58,8 @@ func execute(cmd command, t *testing.T, client *mockHttpClient) (string, string) os.Setenv("VESPA_CLI_CACHE_DIR", cmd.cacheDir) // Reset flags to their default value - persistent flags in Cobra persists over tests + // TODO: Due to the bad design of viper, the only proper fix is to get rid of global state by moving each command to + // their own sub-package rootCmd.Flags().VisitAll(resetFlag) documentCmd.Flags().VisitAll(resetFlag) diff --git a/client/go/cmd/deploy.go b/client/go/cmd/deploy.go index 7c9c3d8cc02..034dac2e67b 100644 --- a/client/go/cmd/deploy.go +++ b/client/go/cmd/deploy.go @@ -62,20 +62,7 @@ $ vespa deploy -t cloud -z perf.aws-us-east-1c`, return } target := getTarget() - opts := vespa.DeploymentOpts{ApplicationPackage: pkg, Target: target} - if opts.IsCloud() { - deployment := deploymentFromArgs() - if !opts.ApplicationPackage.HasCertificate() { - fatalErrHint(fmt.Errorf("Missing certificate in application package"), "Applications in Vespa Cloud require a certificate", "Try 'vespa cert'") - return - } - opts.APIKey, err = cfg.ReadAPIKey(deployment.Application.Tenant) - if err != nil { - fatalErrHint(err, "Deployment to cloud requires an API key. Try 'vespa api-key'") - return - } - opts.Deployment = deployment - } + opts := getDeploymentOpts(cfg, pkg, target) if sessionOrRunID, err := vespa.Deploy(opts); err == nil { if opts.IsCloud() { printSuccess("Triggered deployment of ", color.Cyan(pkg.Path), " with run ID ", color.Cyan(sessionOrRunID)) diff --git a/client/go/cmd/helpers.go b/client/go/cmd/helpers.go index b5525cf11fe..4768290e33e 100644 --- a/client/go/cmd/helpers.go +++ b/client/go/cmd/helpers.go @@ -30,7 +30,9 @@ func fatalErr(err error, msg ...interface{}) { } func printErrHint(err error, hints ...string) { - printErr(nil, err.Error()) + if err != nil { + printErr(nil, err.Error()) + } for _, hint := range hints { fmt.Fprintln(stderr, color.Cyan("Hint:"), hint) } @@ -140,9 +142,10 @@ func getService(service string, sessionOrRunID int64) *vespa.Service { return s } +func getSystem() string { return os.Getenv("VESPA_CLI_CLOUD_SYSTEM") } + func getConsoleURL() string { - system := os.Getenv("VESPA_CLI_CLOUD_SYSTEM") - if system == "publiccd" { + if getSystem() == "publiccd" { return "https://console-cd.vespa.oath.cloud" } return "https://console.vespa.oath.cloud" @@ -150,8 +153,7 @@ func getConsoleURL() string { } func getApiURL() string { - system := os.Getenv("VESPA_CLI_CLOUD_SYSTEM") - if system == "publiccd" { + if getSystem() == "publiccd" { return "https://api.vespa-external-cd.aws.oath.cloud:4443" } return "https://api.vespa-external.aws.oath.cloud:4443" @@ -221,3 +223,22 @@ func waitForService(service string, sessionOrRunID int64) { fatalErr(err, s.Description(), " at ", color.Cyan(s.BaseURL), " is ", color.Red("not ready")) } } + +func getDeploymentOpts(cfg *Config, pkg vespa.ApplicationPackage, target vespa.Target) vespa.DeploymentOpts { + opts := vespa.DeploymentOpts{ApplicationPackage: pkg, Target: target} + if opts.IsCloud() { + deployment := deploymentFromArgs() + if !opts.ApplicationPackage.HasCertificate() { + fatalErrHint(fmt.Errorf("Missing certificate in application package"), "Applications in Vespa Cloud require a certificate", "Try 'vespa cert'") + return opts + } + var err error + opts.APIKey, err = cfg.ReadAPIKey(deployment.Application.Tenant) + if err != nil { + fatalErrHint(err, "Deployment to cloud requires an API key. Try 'vespa api-key'") + return opts + } + opts.Deployment = deployment + } + return opts +} diff --git a/client/go/cmd/prod.go b/client/go/cmd/prod.go index 15d9edc18a7..382ede0fae8 100644 --- a/client/go/cmd/prod.go +++ b/client/go/cmd/prod.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io/ioutil" + "log" "os" "path/filepath" "strings" @@ -20,6 +21,7 @@ import ( func init() { rootCmd.AddCommand(prodCmd) prodCmd.AddCommand(prodInitCmd) + prodCmd.AddCommand(prodSubmitCmd) } var prodCmd = &cobra.Command{ @@ -60,7 +62,7 @@ https://cloud.vespa.ai/en/reference/deployment`, } if pkg.IsZip() { fatalErrHint(fmt.Errorf("Cannot modify compressed application package %s", pkg.Path), - "Try running 'mvn clean' and re-running this command") + "Try running 'mvn clean' and run this command again") return } @@ -95,6 +97,69 @@ https://cloud.vespa.ai/en/reference/deployment`, }, } +var prodSubmitCmd = &cobra.Command{ + Use: "submit", + Short: "Submit your application for production deployment", + Long: `Submit your application for production deployment. + +This commands uploads your application package to Vespa Cloud and deploys it to +the production zones specified in deployment.xml. + +Nodes are allocated to your application according to resources specified in +services.xml. + +While submitting an application from a local development environment is +supported, it's strongly recommended that production deployments are performed +by a continuous build system. + +For more information about production deployments in Vespa Cloud see: +https://cloud.vespa.ai/en/getting-to-production +https://cloud.vespa.ai/en/automated-deployments`, + DisableAutoGenTag: true, + Example: `$ mvn package +$ vespa prod submit`, + Run: func(cmd *cobra.Command, args []string) { + target := getTarget() + if target.Type() != "cloud" { + fatalErr(fmt.Errorf("%s target cannot deploy to Vespa Cloud", target.Type())) + return + } + appSource := applicationSource(args) + pkg, err := vespa.FindApplicationPackage(appSource, true) + if err != nil { + fatalErr(err) + return + } + cfg, err := LoadConfig() + if err != nil { + fatalErr(err, "Could not load config") + return + } + if !pkg.HasDeployment() { + fatalErrHint(fmt.Errorf("No deployment.xml found"), "Try creating one with vespa prod init") + return + } + if !pkg.IsJava() { + // TODO: Loosen this requirement when we start supporting applications with Java in production + fatalErrHint(fmt.Errorf("No jar files found in %s", pkg.Path), "Only applications containing Java components are currently supported") + return + } + isCI := os.Getenv("CI") != "" + if !isCI { + fmt.Fprintln(stderr, color.Yellow("Warning:"), "Submitting from a non-CI environment is discouraged") + printErrHint(nil, "See https://cloud.vespa.ai/en/getting-to-production for best practices") + } + opts := getDeploymentOpts(cfg, pkg, target) + if err := vespa.Submit(opts); err != nil { + fatalErr(err, "Could not submit application for deployment") + } else { + printSuccess("Submitted ", color.Cyan(pkg.Path), " for deployment") + log.Printf("See %s for deployment progress\n", color.Cyan(fmt.Sprintf("%s/tenant/%s/application/%s/prod/deployment", + getConsoleURL(), opts.Deployment.Application.Tenant, opts.Deployment.Application.Application))) + } + }, +} + func writeWithBackup(pkg vespa.ApplicationPackage, filename, contents string) error { dst := filepath.Join(pkg.Path, filename) if util.PathExists(dst) { @@ -133,6 +198,13 @@ func updateRegions(r *bufio.Reader, deploymentXML xml.Deployment) xml.Deployment if err := deploymentXML.Replace("prod", "region", regionElements); err != nil { fatalErr(err, "Could not update region elements in deployment.xml") } + // TODO: Some sample apps come with production <test> elements, but not necessarily working production tests, we + // therefore remove <test> elements here. + // This can be improved by supporting <test> elements in xml package and allow specifying testing as part of + // region prompt, e.g. region1;test,region2 + if err := deploymentXML.Replace("prod", "test", nil); err != nil { + fatalErr(err, "Could not remove test elements in deployment.xml") + } return deploymentXML } @@ -152,7 +224,7 @@ func promptRegions(r *bufio.Reader, deploymentXML xml.Deployment) string { validator := func(input string) error { regions := strings.Split(input, ",") for _, r := range regions { - if !xml.ValidProdRegion(r) { + if !xml.IsProdRegion(r, getSystem()) { return fmt.Errorf("invalid region %s", r) } } diff --git a/client/go/cmd/prod_test.go b/client/go/cmd/prod_test.go index 5abb6d24577..4ce6112122a 100644 --- a/client/go/cmd/prod_test.go +++ b/client/go/cmd/prod_test.go @@ -2,6 +2,7 @@ package cmd import ( "bytes" + "io" "io/ioutil" "os" "path/filepath" @@ -82,6 +83,7 @@ func readFileString(t *testing.T, filename string) string { func createApplication(t *testing.T, pkgDir string) { appDir := filepath.Join(pkgDir, "src", "main", "application") + targetDir := filepath.Join(pkgDir, "target") if err := os.MkdirAll(appDir, 0755); err != nil { t.Fatal(err) } @@ -115,4 +117,50 @@ func createApplication(t *testing.T, pkgDir string) { if err := ioutil.WriteFile(filepath.Join(appDir, "services.xml"), []byte(servicesXML), 0644); err != nil { t.Fatal(err) } + if err := os.MkdirAll(targetDir, 0755); err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(filepath.Join(pkgDir, "pom.xml"), []byte(""), 0644); err != nil { + t.Fatal(err) + } +} + +func TestProdSubmit(t *testing.T) { + homeDir := filepath.Join(t.TempDir(), ".vespa") + pkgDir := filepath.Join(t.TempDir(), "app") + createApplication(t, pkgDir) + + httpClient := &mockHttpClient{} + httpClient.NextResponse(200, `ok`) + execute(command{homeDir: homeDir, args: []string{"config", "set", "application", "t1.a1.i1"}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "cloud"}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"api-key"}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"cert", pkgDir}}, t, httpClient) + + // Copy an application package pre-assambled with mvn package + testAppDir := filepath.Join("testdata", "applications", "withDeployment", "target") + zipFile := filepath.Join(testAppDir, "application.zip") + copyFile(t, filepath.Join(pkgDir, "target", "application.zip"), zipFile) + testZipFile := filepath.Join(testAppDir, "application-test.zip") + copyFile(t, filepath.Join(pkgDir, "target", "application-test.zip"), testZipFile) + + out, _ := execute(command{homeDir: homeDir, args: []string{"prod", "submit", pkgDir}}, t, httpClient) + assert.Contains(t, out, "Success: Submitted") + assert.Contains(t, out, "See https://console.vespa.oath.cloud/tenant/t1/application/a1/prod/deployment for deployment progress") +} + +func copyFile(t *testing.T, dstFilename, srcFilename string) { + dst, err := os.Create(dstFilename) + if err != nil { + t.Fatal(err) + } + defer dst.Close() + src, err := os.Open(srcFilename) + if err != nil { + t.Fatal(err) + } + defer src.Close() + if _, err := io.Copy(dst, src); err != nil { + t.Fatal(err) + } } diff --git a/client/go/cmd/query_test.go b/client/go/cmd/query_test.go index 81dc03766be..ec6c3063906 100644 --- a/client/go/cmd/query_test.go +++ b/client/go/cmd/query_test.go @@ -45,12 +45,12 @@ func TestServerError(t *testing.T) { func assertQuery(t *testing.T, expectedQuery string, query ...string) { client := &mockHttpClient{} - queryURL := queryServiceURL(client) client.NextResponse(200, "{\"query\":\"result\"}") assert.Equal(t, "{\n \"query\": \"result\"\n}\n", executeCommand(t, client, []string{"query"}, query), "query output") + queryURL := queryServiceURL(client) assert.Equal(t, queryURL+"/search/"+expectedQuery, client.lastRequest.URL.String()) } diff --git a/client/go/cmd/testdata/applications/withDeployment/target/application-test.zip b/client/go/cmd/testdata/applications/withDeployment/target/application-test.zip Binary files differnew file mode 100644 index 00000000000..8a1707b9cee --- /dev/null +++ b/client/go/cmd/testdata/applications/withDeployment/target/application-test.zip diff --git a/client/go/cmd/testdata/applications/withDeployment/target/application.zip b/client/go/cmd/testdata/applications/withDeployment/target/application.zip Binary files differnew file mode 100644 index 00000000000..da23c2ff437 --- /dev/null +++ b/client/go/cmd/testdata/applications/withDeployment/target/application.zip |