diff options
author | Martin Polden <mpolden@mpolden.no> | 2023-11-22 10:32:14 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-11-22 10:32:14 +0100 |
commit | 07e8c4bee1bffd940bdc54bc093566f3d78961b7 (patch) | |
tree | 6f8a7b2662d1ce63e8e688d449fbfe0717163a2d /client | |
parent | 623dee3517d0151a0121cbaebb123f415cc77de9 (diff) | |
parent | 2bc23cc885ab703acf762191ac8114451cac82b9 (diff) |
Merge pull request #29410 from vespa-engine/mpolden/fetch-pkg
Add command for downloading application package
Diffstat (limited to 'client')
-rw-r--r-- | client/go/internal/cli/cmd/deploy.go | 4 | ||||
-rw-r--r-- | client/go/internal/cli/cmd/fetch.go | 46 | ||||
-rw-r--r-- | client/go/internal/cli/cmd/root.go | 1 | ||||
-rw-r--r-- | client/go/internal/cli/cmd/visit.go | 4 | ||||
-rw-r--r-- | client/go/internal/vespa/deploy.go | 123 | ||||
-rw-r--r-- | client/go/internal/vespa/deploy_test.go | 66 |
6 files changed, 239 insertions, 5 deletions
diff --git a/client/go/internal/cli/cmd/deploy.go b/client/go/internal/cli/cmd/deploy.go index 8806a21c9fc..dd605237b5f 100644 --- a/client/go/internal/cli/cmd/deploy.go +++ b/client/go/internal/cli/cmd/deploy.go @@ -60,7 +60,7 @@ $ vespa deploy -t cloud -z perf.aws-us-east-1c`, return err } timeout := time.Duration(waitSecs) * time.Second - opts := vespa.DeploymentOptions{ApplicationPackage: pkg, Target: target, Timeout: timeout} + opts := vespa.DeploymentOptions{ApplicationPackage: pkg, Target: target} if versionArg != "" { version, err := version.Parse(versionArg) if err != nil { @@ -162,7 +162,7 @@ func newActivateCmd(cli *CLI) *cobra.Command { if _, err := waiter.DeployService(target); err != nil { return err } - opts := vespa.DeploymentOptions{Target: target, Timeout: timeout} + opts := vespa.DeploymentOptions{Target: target} err = vespa.Activate(sessionID, opts) if err != nil { return err diff --git a/client/go/internal/cli/cmd/fetch.go b/client/go/internal/cli/cmd/fetch.go new file mode 100644 index 00000000000..b2e7d11ba7b --- /dev/null +++ b/client/go/internal/cli/cmd/fetch.go @@ -0,0 +1,46 @@ +package cmd + +import ( + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func newFetchCmd(cli *CLI) *cobra.Command { + cmd := &cobra.Command{ + Use: "fetch [path]", + Short: "Download a deployed application package", + Long: `Download a deployed application package. + +This command can be used to download an already deployed Vespa application +package. The package is written as a ZIP file to the given path, or current +directory if no path is given. +`, + Example: `$ vespa fetch +$ vespa fetch mydir/ +$ vespa fetch -t cloud mycloudapp.zip +`, + Args: cobra.MaximumNArgs(1), + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + target, err := cli.target(targetOptions{}) + if err != nil { + return err + } + path := "." + if len(args) > 0 { + path = args[0] + } + dstPath := "" + if err := cli.spinner(cli.Stderr, "Downloading application package...", func() error { + dstPath, err = vespa.Fetch(vespa.DeploymentOptions{Target: target}, path) + return err + }); err != nil { + return err + } + cli.printSuccess("Application package written to ", dstPath) + return nil + }, + } + return cmd +} diff --git a/client/go/internal/cli/cmd/root.go b/client/go/internal/cli/cmd/root.go index 4d1a7cf6f89..004bdc038fe 100644 --- a/client/go/internal/cli/cmd/root.go +++ b/client/go/internal/cli/cmd/root.go @@ -282,6 +282,7 @@ func (c *CLI) configureCommands() { rootCmd.AddCommand(newVersionCmd(c)) // version rootCmd.AddCommand(newVisitCmd(c)) // visit rootCmd.AddCommand(newFeedCmd(c)) // feed + rootCmd.AddCommand(newFetchCmd(c)) // fetch } func (c *CLI) bindWaitFlag(cmd *cobra.Command, defaultSecs int, value *int) { diff --git a/client/go/internal/cli/cmd/visit.go b/client/go/internal/cli/cmd/visit.go index bb226701e0a..2ca01764deb 100644 --- a/client/go/internal/cli/cmd/visit.go +++ b/client/go/internal/cli/cmd/visit.go @@ -90,8 +90,8 @@ func newVisitCmd(cli *CLI) *cobra.Command { ) cmd := &cobra.Command{ Use: "visit", - Short: "Fetch and print all documents from Vespa", - Long: `Fetch and print all documents from Vespa. + Short: "Retrieve and print all documents from Vespa", + Long: `Retrieve and print all documents from Vespa. By default prints each document received on its own line (JSONL format). `, diff --git a/client/go/internal/vespa/deploy.go b/client/go/internal/vespa/deploy.go index b39c51916e7..4684e313291 100644 --- a/client/go/internal/vespa/deploy.go +++ b/client/go/internal/vespa/deploy.go @@ -12,6 +12,7 @@ import ( "mime/multipart" "net/http" "net/url" + "os" "path/filepath" "strconv" "strings" @@ -47,7 +48,6 @@ type Deployment struct { type DeploymentOptions struct { Target Target ApplicationPackage ApplicationPackage - Timeout time.Duration Version version.Version } @@ -119,6 +119,127 @@ func ZoneFromString(s string) (ZoneID, error) { return ZoneID{Environment: parts[0], Region: parts[1]}, nil } +func Fetch(deployment DeploymentOptions, path string) (string, error) { + if util.IsDirectory(path) { + path = filepath.Join(path, "application.zip") + } + if util.PathExists(path) { + return "", fmt.Errorf("%s already exists", path) + } + if deployment.Target.IsCloud() { + return path, fetchFromController(deployment, path) + } + return path, fetchFromConfigServer(deployment, path) +} + +func deployServiceGet(url string, deployment DeploymentOptions, w io.Writer) error { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } + response, err := deployServiceDo(req, 0, deployment) + if err != nil { + return err + } + defer response.Body.Close() + _, err = io.Copy(w, response.Body) + return err +} + +func fetchFromController(deployment DeploymentOptions, path string) error { + var ( + pkgURL *url.URL + err error + ) + switch deployment.Target.Deployment().Zone.Environment { + case "dev", "perf": + pkgURL, err = deployment.url(fmt.Sprintf("/application/v4/tenant/%s/application/%s/instance/%s/job/%s/package", + deployment.Target.Deployment().Application.Tenant, + deployment.Target.Deployment().Application.Application, + deployment.Target.Deployment().Application.Instance, + deployment.Target.Deployment().Zone.Environment+"-"+deployment.Target.Deployment().Zone.Region, + )) + default: + pkgURL, err = deployment.url(fmt.Sprintf("/application/v4/tenant/%s/application/%s/package", + deployment.Target.Deployment().Application.Tenant, + deployment.Target.Deployment().Application.Application), + ) + } + if err != nil { + return err + } + tmpFile, err := os.CreateTemp("", "vespa") + if err != nil { + return err + } + defer os.Remove(tmpFile.Name()) + if err := deployServiceGet(pkgURL.String(), deployment, tmpFile); err != nil { + return err + } + if err := tmpFile.Close(); err != nil { + return err + } + return os.Rename(tmpFile.Name(), path) +} + +func fetchFromConfigServer(deployment DeploymentOptions, path string) error { + tmpDir, err := os.MkdirTemp("", "vespa") + if err != nil { + return err + } + defer os.RemoveAll(tmpDir) + u, err := deployment.url("/application/v2/tenant/default/application/default/environment/prod/region/default/instance/default/content") + if err != nil { + return err + } + dir := filepath.Join(tmpDir, "application") + if err := fetchFilesFromConfigServer(deployment, u, dir); err != nil { + return err + } + zipFile := filepath.Join(tmpDir, "application.zip") + if err := zipDir(dir, zipFile); err != nil { + return err + } + return os.Rename(zipFile, path) +} + +func fetchFilesFromConfigServer(deployment DeploymentOptions, contentURL *url.URL, path string) error { + var data bytes.Buffer + if err := deployServiceGet(contentURL.String(), deployment, &data); err != nil { + return err + } + var fileURLs []string + if err := json.Unmarshal(data.Bytes(), &fileURLs); err != nil { + return err + } + for _, fu := range fileURLs { + u, err := url.Parse(fu) + if err != nil { + return err + } + entryName := filepath.Join(path, filepath.Base(u.Path)) + if strings.HasSuffix(u.Path, "/") { + if err := fetchFilesFromConfigServer(deployment, u, entryName); err != nil { + return err + } + } else { + if err := os.MkdirAll(filepath.Dir(entryName), 0755); err != nil { + return err + } + f, err := os.Create(entryName) + if err != nil { + return err + } + if err := deployServiceGet(fu, deployment, f); err != nil { + f.Close() + return err + } + f.Close() + } + } + return nil +} + // Prepare deployment and return the session ID func Prepare(deployment DeploymentOptions) (PrepareResult, error) { if deployment.Target.IsCloud() { diff --git a/client/go/internal/vespa/deploy_test.go b/client/go/internal/vespa/deploy_test.go index 9dfdc47d8e6..09129d3027a 100644 --- a/client/go/internal/vespa/deploy_test.go +++ b/client/go/internal/vespa/deploy_test.go @@ -2,6 +2,7 @@ package vespa import ( + "archive/zip" "io" "mime" "mime/multipart" @@ -14,6 +15,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vespa-engine/vespa/client/go/internal/mock" + "github.com/vespa-engine/vespa/client/go/internal/util" "github.com/vespa-engine/vespa/client/go/internal/version" ) @@ -196,6 +198,70 @@ func TestDeactivateCloud(t *testing.T) { assert.Equal(t, "https://api-ctl.vespa-cloud.com:4443/application/v4/tenant/t1/application/a1/instance/i1/environment/dev/region/us-north-1", req.URL.String()) } +func TestFetch(t *testing.T) { + httpClient := mock.HTTPClient{} + target := LocalTarget(&httpClient, TLSOptions{}, 0) + opts := DeploymentOptions{Target: target} + httpClient.NextResponse(mock.HTTPResponse{ + URI: "/application/v2/tenant/default/application/default/environment/prod/region/default/instance/default/content", + Status: 200, + Body: []byte(`[ +"/application/v2/tenant/default/application/default/environment/prod/region/default/instance/default/content/schemas/", +"/application/v2/tenant/default/application/default/environment/prod/region/default/instance/default/content/services.xml" +]`), + }) + httpClient.NextResponse(mock.HTTPResponse{ + URI: "/application/v2/tenant/default/application/default/environment/prod/region/default/instance/default/content/schemas/", + Status: 200, + Body: []byte(`[ +"/application/v2/tenant/default/application/default/environment/prod/region/default/instance/default/content/schemas/music.sd" +]`), + }) + httpClient.NextResponse(mock.HTTPResponse{ + URI: "/application/v2/tenant/default/application/default/environment/prod/region/default/instance/default/content/schemas/music.sd", + Status: 200, + Body: []byte(`music.sd contents`), + }) + httpClient.NextResponse(mock.HTTPResponse{ + URI: "/application/v2/tenant/default/application/default/environment/prod/region/default/instance/default/content/services.xml", + Status: 200, + Body: []byte(`services.xml contents`), + }) + dir := t.TempDir() + dst, err := Fetch(opts, dir) + require.Nil(t, err) + assert.True(t, util.PathExists(dst)) + + f, err := os.Open(dst) + require.Nil(t, err) + defer f.Close() + zr, err := zip.NewReader(f, 1000) + require.Nil(t, err) + schema, err := zr.Open("schemas/music.sd") + require.Nil(t, err) + data, err := io.ReadAll(schema) + require.Nil(t, err) + assert.Equal(t, `music.sd contents`, string(data)) +} + +func TestFetchCloud(t *testing.T) { + httpClient := mock.HTTPClient{} + target, _ := createCloudTarget(t, io.Discard) + cloudTarget, ok := target.(*cloudTarget) + require.True(t, ok) + cloudTarget.httpClient = &httpClient + opts := DeploymentOptions{Target: target} + httpClient.NextResponse(mock.HTTPResponse{ + URI: "/application/v4/tenant/t1/application/a1/instance/i1/job/dev-us-north-1/package", + Status: 200, + Body: []byte(`application zip`), + }) + dir := t.TempDir() + dst, err := Fetch(opts, dir) + require.Nil(t, err) + assert.True(t, util.PathExists(dst)) +} + type pkgFixture struct { expectedPath string expectedTestPath string |