diff options
author | Martin Polden <mpolden@mpolden.no> | 2023-02-03 15:20:23 +0100 |
---|---|---|
committer | Martin Polden <mpolden@mpolden.no> | 2023-02-03 15:35:25 +0100 |
commit | e1e94812425a487069bf33f781bec987e9e49874 (patch) | |
tree | 4a892c3b5c0a7dee2cb76f9971e538cb4aba8a16 /client/go/internal/cli | |
parent | a08ae588d6035b69f0961dff596fc871fd1c4e58 (diff) |
Re-organize Go code
Diffstat (limited to 'client/go/internal/cli')
107 files changed, 10518 insertions, 0 deletions
diff --git a/client/go/internal/cli/auth/auth.go b/client/go/internal/cli/auth/auth.go new file mode 100644 index 00000000000..100af7ea1d2 --- /dev/null +++ b/client/go/internal/cli/auth/auth.go @@ -0,0 +1,139 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package auth + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "time" +) + +const ( + waitThresholdInSeconds = 3 + // SecretsNamespace namespace used to set/get values from the keychain + SecretsNamespace = "vespa-cli" +) + +var requiredScopes = []string{"openid", "offline_access"} + +type Authenticator struct { + Audience string + ClientID string + DeviceCodeEndpoint string + OauthTokenEndpoint string +} + +// SecretStore provides access to stored sensitive data. +type SecretStore interface { + // Get gets the secret + Get(namespace, key string) (string, error) + // Delete removes the secret + Delete(namespace, key string) error +} + +type Result struct { + RefreshToken string + AccessToken string + ExpiresIn int64 +} + +type State struct { + DeviceCode string `json:"device_code"` + UserCode string `json:"user_code"` + VerificationURI string `json:"verification_uri_complete"` + ExpiresIn int `json:"expires_in"` + Interval int `json:"interval"` +} + +// RequiredScopes returns the scopes used for login. +func RequiredScopes() []string { return requiredScopes } + +func (s *State) IntervalDuration() time.Duration { + return time.Duration(s.Interval+waitThresholdInSeconds) * time.Second +} + +// Start kicks-off the device authentication flow +// by requesting a device code from Auth0, +// The returned state contains the URI for the next step of the flow. +func (a *Authenticator) Start(ctx context.Context) (State, error) { + s, err := a.getDeviceCode(ctx) + if err != nil { + return State{}, fmt.Errorf("cannot get device code: %w", err) + } + return s, nil +} + +// Wait waits until the user is logged in on the browser. +func (a *Authenticator) Wait(ctx context.Context, state State) (Result, error) { + t := time.NewTicker(state.IntervalDuration()) + for { + select { + case <-ctx.Done(): + return Result{}, ctx.Err() + case <-t.C: + data := url.Values{ + "client_id": {a.ClientID}, + "grant_type": {"urn:ietf:params:oauth:grant-type:device_code"}, + "device_code": {state.DeviceCode}, + } + r, err := http.PostForm(a.OauthTokenEndpoint, data) + if err != nil { + return Result{}, fmt.Errorf("cannot get device code: %w", err) + } + defer r.Body.Close() + + var res struct { + AccessToken string `json:"access_token"` + IDToken string `json:"id_token"` + RefreshToken string `json:"refresh_token"` + Scope string `json:"scope"` + ExpiresIn int64 `json:"expires_in"` + TokenType string `json:"token_type"` + Error *string `json:"error,omitempty"` + ErrorDescription string `json:"error_description,omitempty"` + } + + err = json.NewDecoder(r.Body).Decode(&res) + if err != nil { + return Result{}, fmt.Errorf("cannot decode response: %w", err) + } + + if res.Error != nil { + if *res.Error == "authorization_pending" { + continue + } + return Result{}, errors.New(res.ErrorDescription) + } + + return Result{ + RefreshToken: res.RefreshToken, + AccessToken: res.AccessToken, + ExpiresIn: res.ExpiresIn, + }, nil + } + } +} + +func (a *Authenticator) getDeviceCode(ctx context.Context) (State, error) { + data := url.Values{ + "client_id": {a.ClientID}, + "scope": {strings.Join(requiredScopes, " ")}, + "audience": {a.Audience}, + } + r, err := http.PostForm(a.DeviceCodeEndpoint, data) + if err != nil { + return State{}, fmt.Errorf("cannot get device code: %w", err) + } + defer r.Body.Close() + var res State + err = json.NewDecoder(r.Body).Decode(&res) + if err != nil { + return State{}, fmt.Errorf("cannot decode response: %w", err) + } + return res, nil +} diff --git a/client/go/internal/cli/auth/auth0/auth0.go b/client/go/internal/cli/auth/auth0/auth0.go new file mode 100644 index 00000000000..36dc0b8871c --- /dev/null +++ b/client/go/internal/cli/auth/auth0/auth0.go @@ -0,0 +1,247 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package auth0 + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "os/signal" + "path/filepath" + "sort" + "time" + + "github.com/vespa-engine/vespa/client/go/internal/cli/auth" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +const ( + accessTokenExpiry = 5 * time.Minute + reauthMessage = "re-authenticate with 'vespa auth login'" +) + +// Credentials holds the credentials retrieved from Auth0. +type Credentials struct { + AccessToken string `json:"access_token,omitempty"` + Scopes []string `json:"scopes,omitempty"` + ExpiresAt time.Time `json:"expires_at"` +} + +// Client is a client for the Auth0 service. +type Client struct { + httpClient util.HTTPClient + Authenticator *auth.Authenticator // TODO: Make this private + configPath string + systemName string + systemURL string + provider auth0Provider +} + +// config is the root type of the persisted config +type config struct { + Version int `json:"version"` + Providers providers `json:"providers"` +} + +type providers struct { + Auth0 auth0Provider `json:"auth0"` +} + +type auth0Provider struct { + Version int `json:"version"` + Systems map[string]Credentials `json:"systems"` +} + +// flowConfig represents the authorization flow configuration retrieved from a Vespa system. +type flowConfig struct { + Audience string `json:"audience"` + ClientID string `json:"client-id"` + DeviceCodeEndpoint string `json:"device-code-endpoint"` + OauthTokenEndpoint string `json:"oauth-token-endpoint"` +} + +func cancelOnInterrupt() context.Context { + ctx, cancel := context.WithCancel(context.Background()) + ch := make(chan os.Signal, 1) + signal.Notify(ch, os.Interrupt) + go func() { + <-ch + defer cancel() + os.Exit(0) + }() + return ctx +} + +func newClient(httpClient util.HTTPClient, configPath, systemName, systemURL string) (*Client, error) { + a := Client{} + a.httpClient = httpClient + a.configPath = configPath + a.systemName = systemName + a.systemURL = systemURL + c, err := a.getDeviceFlowConfig() + if err != nil { + return nil, err + } + a.Authenticator = &auth.Authenticator{ + Audience: c.Audience, + ClientID: c.ClientID, + DeviceCodeEndpoint: c.DeviceCodeEndpoint, + OauthTokenEndpoint: c.OauthTokenEndpoint, + } + provider, err := readConfig(configPath) + if err != nil { + return nil, err + } + a.provider = provider + return &a, nil +} + +// New constructs a new Auth0 client, storing configuration in the given configPath. The client will be configured for +// use in the given Vespa system. +func New(configPath string, systemName, systemURL string) (*Client, error) { + return newClient(util.CreateClient(time.Second*30), configPath, systemName, systemURL) +} + +func (a *Client) getDeviceFlowConfig() (flowConfig, error) { + url := a.systemURL + "/auth0/v1/device-flow-config" + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return flowConfig{}, err + } + r, err := a.httpClient.Do(req, time.Second*30) + if err != nil { + return flowConfig{}, fmt.Errorf("failed to get device flow config: %w", err) + } + defer r.Body.Close() + if r.StatusCode/100 != 2 { + return flowConfig{}, fmt.Errorf("failed to get device flow config: got response code %d from %s", r.StatusCode, url) + } + var cfg flowConfig + if err := json.NewDecoder(r.Body).Decode(&cfg); err != nil { + return flowConfig{}, fmt.Errorf("failed to decode response: %w", err) + } + return cfg, nil +} + +// GetAccessToken returns an access token for the configured system, refreshing it if necessary. +func (a *Client) GetAccessToken() (string, error) { + creds, ok := a.provider.Systems[a.systemName] + if !ok { + return "", fmt.Errorf("system %s is not configured", a.systemName) + } else if creds.AccessToken == "" { + return "", fmt.Errorf("access token missing: %s", reauthMessage) + } else if scopesChanged(creds) { + return "", fmt.Errorf("authentication scopes changed: %s", reauthMessage) + } else if isExpired(creds.ExpiresAt, accessTokenExpiry) { + // check if the stored access token is expired: + // use the refresh token to get a new access token: + tr := &auth.TokenRetriever{ + Authenticator: a.Authenticator, + Secrets: &auth.Keyring{}, + Client: http.DefaultClient, + } + resp, err := tr.Refresh(cancelOnInterrupt(), a.systemName) + if err != nil { + return "", fmt.Errorf("failed to renew access token: %w: %s", err, reauthMessage) + } else { + // persist the updated system with renewed access token + creds.AccessToken = resp.AccessToken + creds.ExpiresAt = time.Now().Add(time.Duration(resp.ExpiresIn) * time.Second) + if err := a.WriteCredentials(creds); err != nil { + return "", err + } + } + } + return creds.AccessToken, nil +} + +func isExpired(t time.Time, ttl time.Duration) bool { return time.Now().Add(ttl).After(t) } + +func scopesChanged(s Credentials) bool { + required := auth.RequiredScopes() + current := s.Scopes + if len(required) != len(current) { + return true + } + sort.Strings(required) + sort.Strings(current) + for i := range s.Scopes { + if required[i] != current[i] { + return true + } + } + return false +} + +// HasCredentials returns true if this client has retrived credentials for the configured system. +func (a *Client) HasCredentials() bool { + _, ok := a.provider.Systems[a.systemName] + return ok +} + +// WriteCredentials writes given credentials to the configuration file. +func (a *Client) WriteCredentials(credentials Credentials) error { + if a.provider.Systems == nil { + a.provider.Systems = make(map[string]Credentials) + } + a.provider.Systems[a.systemName] = credentials + if err := writeConfig(a.provider, a.configPath); err != nil { + return fmt.Errorf("failed to write config: %w", err) + } + return nil +} + +// RemoveCredentials removes credentials for the system configured in this client. +func (a *Client) RemoveCredentials() error { + tr := &auth.TokenRetriever{Secrets: &auth.Keyring{}} + if err := tr.Delete(a.systemName); err != nil { + return fmt.Errorf("failed to remove system %s from secret storage: %w", a.systemName, err) + } + delete(a.provider.Systems, a.systemName) + if err := writeConfig(a.provider, a.configPath); err != nil { + return fmt.Errorf("failed to write config: %w", err) + } + return nil +} + +func writeConfig(provider auth0Provider, path string) error { + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0700); err != nil { + return err + } + version := 1 + provider.Version = version + r := config{ + Version: version, + Providers: providers{ + Auth0: provider, + }, + } + jsonConfig, err := json.MarshalIndent(r, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, jsonConfig, 0600) +} + +func readConfig(path string) (auth0Provider, error) { + f, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return auth0Provider{}, nil + } + return auth0Provider{}, err + } + defer f.Close() + cfg := config{} + if err := json.NewDecoder(f).Decode(&cfg); err != nil { + return auth0Provider{}, err + } + auth0Provider := cfg.Providers.Auth0 + if auth0Provider.Systems == nil { + auth0Provider.Systems = make(map[string]Credentials) + } + return auth0Provider, nil +} diff --git a/client/go/internal/cli/auth/auth0/auth0_test.go b/client/go/internal/cli/auth/auth0/auth0_test.go new file mode 100644 index 00000000000..39393bbdfc1 --- /dev/null +++ b/client/go/internal/cli/auth/auth0/auth0_test.go @@ -0,0 +1,99 @@ +package auth0 + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vespa-engine/vespa/client/go/internal/mock" +) + +func TestConfigWriting(t *testing.T) { + configPath := filepath.Join(t.TempDir(), "config") + httpClient := mock.HTTPClient{} + flowConfigResponse := `{ + "audience": "https://example.com/api/v2/", + "client-id": "some-id", + "device-code-endpoint": "https://example.com/oauth/device/code", + "oauth-token-endpoint": "https://example.com/oauth/token" +}` + httpClient.NextResponseString(200, flowConfigResponse) + client, err := newClient(&httpClient, configPath, "public", "http://example.com") + require.Nil(t, err) + assert.Equal(t, "https://example.com/api/v2/", client.Authenticator.Audience) + assert.Equal(t, "some-id", client.Authenticator.ClientID) + assert.Equal(t, "https://example.com/oauth/device/code", client.Authenticator.DeviceCodeEndpoint) + assert.Equal(t, "https://example.com/oauth/token", client.Authenticator.OauthTokenEndpoint) + + creds1 := Credentials{ + AccessToken: "some-token", + Scopes: []string{"foo", "bar"}, + ExpiresAt: time.Date(2022, 03, 01, 15, 45, 50, 0, time.UTC), + } + require.Nil(t, client.WriteCredentials(creds1)) + expected := `{ + "version": 1, + "providers": { + "auth0": { + "version": 1, + "systems": { + "public": { + "access_token": "some-token", + "scopes": [ + "foo", + "bar" + ], + "expires_at": "2022-03-01T15:45:50Z" + } + } + } + } +}` + assertConfig(t, expected, configPath) + + // Switch to another system + httpClient.NextResponseString(200, flowConfigResponse) + client, err = newClient(&httpClient, configPath, "publiccd", "http://example.com") + require.Nil(t, err) + creds2 := Credentials{ + AccessToken: "another-token", + Scopes: []string{"baz"}, + ExpiresAt: time.Date(2022, 03, 01, 15, 45, 50, 0, time.UTC), + } + require.Nil(t, client.WriteCredentials(creds2)) + expected = `{ + "version": 1, + "providers": { + "auth0": { + "version": 1, + "systems": { + "public": { + "access_token": "some-token", + "scopes": [ + "foo", + "bar" + ], + "expires_at": "2022-03-01T15:45:50Z" + }, + "publiccd": { + "access_token": "another-token", + "scopes": [ + "baz" + ], + "expires_at": "2022-03-01T15:45:50Z" + } + } + } + } +}` + assertConfig(t, expected, configPath) +} + +func assertConfig(t *testing.T, expected, path string) { + data, err := os.ReadFile(path) + require.Nil(t, err) + assert.Equal(t, expected, string(data)) +} diff --git a/client/go/internal/cli/auth/secrets.go b/client/go/internal/cli/auth/secrets.go new file mode 100644 index 00000000000..e38d8c56595 --- /dev/null +++ b/client/go/internal/cli/auth/secrets.go @@ -0,0 +1,24 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package auth + +import ( + "github.com/zalando/go-keyring" +) + +type Keyring struct{} + +// Set sets the given key/value pair with the given namespace. +func (k *Keyring) Set(namespace, key, value string) error { + return keyring.Set(namespace, key, value) +} + +// Get gets a value for the given namespace and key. +func (k *Keyring) Get(namespace, key string) (string, error) { + return keyring.Get(namespace, key) +} + +// Delete deletes a value for the given namespace and key. +func (k *Keyring) Delete(namespace, key string) error { + return keyring.Delete(namespace, key) +} diff --git a/client/go/internal/cli/auth/token.go b/client/go/internal/cli/auth/token.go new file mode 100644 index 00000000000..d6f5e6dfa43 --- /dev/null +++ b/client/go/internal/cli/auth/token.go @@ -0,0 +1,74 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package auth + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" +) + +type TokenResponse struct { + AccessToken string `json:"access_token"` + IDToken string `json:"id_token"` + TokenType string `json:"token_type"` + ExpiresIn int `json:"expires_in"` +} + +type TokenRetriever struct { + Authenticator *Authenticator + Secrets SecretStore + Client *http.Client +} + +// Delete deletes the given system from the secrets' storage. +func (t *TokenRetriever) Delete(system string) error { + return t.Secrets.Delete(SecretsNamespace, system) +} + +// Refresh gets a new access token from the provided refresh token, +// The request is used the default client_id and endpoint for device authentication. +func (t *TokenRetriever) Refresh(ctx context.Context, system string) (TokenResponse, error) { + // get stored refresh token: + refreshToken, err := t.Secrets.Get(SecretsNamespace, system) + if err != nil { + return TokenResponse{}, fmt.Errorf("cannot get the stored refresh token: %w", err) + } + if refreshToken == "" { + return TokenResponse{}, errors.New("cannot use the stored refresh token: the token is empty") + } + // get access token: + r, err := t.Client.PostForm(t.Authenticator.OauthTokenEndpoint, url.Values{ + "grant_type": {"refresh_token"}, + "client_id": {t.Authenticator.ClientID}, + "refresh_token": {refreshToken}, + }) + if err != nil { + return TokenResponse{}, fmt.Errorf("cannot get a new access token from the refresh token: %w", err) + } + + defer r.Body.Close() + if r.StatusCode != http.StatusOK { + b, _ := io.ReadAll(r.Body) + res := struct { + Description string `json:"error_description"` + }{} + if json.Unmarshal(b, &res) == nil { + return TokenResponse{}, errors.New(strings.ToLower(strings.TrimSuffix(res.Description, "."))) + } + return TokenResponse{}, fmt.Errorf("cannot get a new access token from the refresh token: %s", string(b)) + } + + var res TokenResponse + err = json.NewDecoder(r.Body).Decode(&res) + if err != nil { + return TokenResponse{}, fmt.Errorf("cannot decode response: %w", err) + } + + return res, nil +} diff --git a/client/go/internal/cli/auth/zts/zts.go b/client/go/internal/cli/auth/zts/zts.go new file mode 100644 index 00000000000..9fabe219209 --- /dev/null +++ b/client/go/internal/cli/auth/zts/zts.go @@ -0,0 +1,58 @@ +package zts + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +const DefaultURL = "https://zts.athenz.ouroath.com:4443" + +// Client is a client for Athenz ZTS, an authentication token service. +type Client struct { + client util.HTTPClient + tokenURL *url.URL +} + +// NewClient creates a new client for an Athenz ZTS service located at serviceURL. +func NewClient(serviceURL string, client util.HTTPClient) (*Client, error) { + tokenURL, err := url.Parse(serviceURL) + if err != nil { + return nil, err + } + tokenURL.Path = "/zts/v1/oauth2/token" + return &Client{tokenURL: tokenURL, client: client}, nil +} + +// AccessToken returns an access token within the given domain, using certificate to authenticate with ZTS. +func (c *Client) AccessToken(domain string, certificate tls.Certificate) (string, error) { + data := fmt.Sprintf("grant_type=client_credentials&scope=%s:domain", domain) + req, err := http.NewRequest("POST", c.tokenURL.String(), strings.NewReader(data)) + if err != nil { + return "", err + } + c.client.UseCertificate([]tls.Certificate{certificate}) + response, err := c.client.Do(req, 10*time.Second) + if err != nil { + return "", err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + return "", fmt.Errorf("got status %d from %s", response.StatusCode, c.tokenURL.String()) + } + var ztsResponse struct { + AccessToken string `json:"access_token"` + } + dec := json.NewDecoder(response.Body) + if err := dec.Decode(&ztsResponse); err != nil { + return "", err + } + return ztsResponse.AccessToken, nil +} diff --git a/client/go/internal/cli/auth/zts/zts_test.go b/client/go/internal/cli/auth/zts/zts_test.go new file mode 100644 index 00000000000..6c6ced9bb33 --- /dev/null +++ b/client/go/internal/cli/auth/zts/zts_test.go @@ -0,0 +1,30 @@ +package zts + +import ( + "crypto/tls" + "testing" + + "github.com/vespa-engine/vespa/client/go/internal/mock" +) + +func TestAccessToken(t *testing.T) { + httpClient := mock.HTTPClient{} + client, err := NewClient("http://example.com", &httpClient) + if err != nil { + t.Fatal(err) + } + httpClient.NextResponseString(400, `{"message": "bad request"}`) + _, err = client.AccessToken("vespa.vespa", tls.Certificate{}) + if err == nil { + t.Fatal("want error for non-ok response status") + } + httpClient.NextResponseString(200, `{"access_token": "foo bar"}`) + token, err := client.AccessToken("vespa.vespa", tls.Certificate{}) + if err != nil { + t.Fatal(err) + } + want := "foo bar" + if token != want { + t.Errorf("got %q, want %q", token, want) + } +} diff --git a/client/go/internal/cli/build/build.go b/client/go/internal/cli/build/build.go new file mode 100644 index 00000000000..a8342a9fb1e --- /dev/null +++ b/client/go/internal/cli/build/build.go @@ -0,0 +1,4 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package build + +var Version string = "0.0.0-devel" // Overriden by linker flag as part of build diff --git a/client/go/internal/cli/cmd/api_key.go b/client/go/internal/cli/cmd/api_key.go new file mode 100644 index 00000000000..367a515f3c3 --- /dev/null +++ b/client/go/internal/cli/cmd/api_key.go @@ -0,0 +1,111 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa api-key command +// Author: mpolden +package cmd + +import ( + "fmt" + "log" + "os" + + "github.com/fatih/color" + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func newAPIKeyCmd(cli *CLI) *cobra.Command { + var overwriteKey bool + cmd := &cobra.Command{ + Use: "api-key", + Short: "Create a new user API key for control-plane authentication with Vespa Cloud", + Long: `Create a new user API key for control-plane authentication with Vespa Cloud. + +The API key will be stored in the Vespa CLI home directory +(see 'vespa help config'). Other commands will then automatically load the API +key as necessary. + +It's possible to override the API key used through environment variables. This +can be useful in continuous integration systems. + +Example of setting the key in-line: + + export VESPA_CLI_API_KEY="my api key" + +Example of loading the key from a custom path: + + export VESPA_CLI_API_KEY_FILE=/path/to/api-key + +Note that when overriding API key through environment variables, that key will +always be used. It's not possible to specify a tenant-specific key. + +Read more in https://cloud.vespa.ai/en/security/guide`, + Example: "$ vespa auth api-key -a my-tenant.my-app.my-instance", + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + return doApiKey(cli, overwriteKey, args) + }, + } + cmd.Flags().BoolVarP(&overwriteKey, "force", "f", false, "Force overwrite of existing API key") + cmd.MarkPersistentFlagRequired(applicationFlag) + return cmd +} + +func doApiKey(cli *CLI, overwriteKey bool, args []string) error { + app, err := cli.config.application() + if err != nil { + return err + } + targetType, err := cli.config.targetType() + if err != nil { + return err + } + system, err := cli.system(targetType) + if err != nil { + return err + } + apiKeyFile := cli.config.apiKeyPath(app.Tenant) + if util.PathExists(apiKeyFile) && !overwriteKey { + err := fmt.Errorf("refusing to overwrite %s", apiKeyFile) + cli.printErr(err, "Use -f to overwrite it") + printPublicKey(system, apiKeyFile, app.Tenant) + return ErrCLI{error: err, quiet: true} + } + apiKey, err := vespa.CreateAPIKey() + if err != nil { + return fmt.Errorf("could not create api key: %w", err) + } + if err := os.WriteFile(apiKeyFile, apiKey, 0600); err == nil { + cli.printSuccess("API private key written to ", apiKeyFile) + return printPublicKey(system, apiKeyFile, app.Tenant) + } else { + return fmt.Errorf("failed to write: %s: %w", apiKeyFile, err) + } +} + +func printPublicKey(system vespa.System, apiKeyFile, tenant string) error { + pemKeyData, err := os.ReadFile(apiKeyFile) + if err != nil { + return fmt.Errorf("failed to read: %s: %w", apiKeyFile, err) + } + key, err := vespa.ECPrivateKeyFrom(pemKeyData) + if err != nil { + return fmt.Errorf("failed to load key: %w", err) + } + pemPublicKey, err := vespa.PEMPublicKeyFrom(key) + if err != nil { + return fmt.Errorf("failed to extract public key: %w", err) + } + fingerprint, err := vespa.FingerprintMD5(pemPublicKey) + if err != nil { + return fmt.Errorf("failed to extract fingerprint: %w", err) + } + log.Printf("\nThis is your public key:\n%s", color.GreenString(string(pemPublicKey))) + log.Printf("Its fingerprint is:\n%s\n", color.CyanString(fingerprint)) + log.Print("\nTo use this key in Vespa Cloud click 'Add custom key' at") + log.Printf(color.CyanString("%s/tenant/%s/account/keys"), system.ConsoleURL, tenant) + log.Print("and paste the entire public key including the BEGIN and END lines.") + return nil +} diff --git a/client/go/internal/cli/cmd/api_key_test.go b/client/go/internal/cli/cmd/api_key_test.go new file mode 100644 index 00000000000..9c14033f85b --- /dev/null +++ b/client/go/internal/cli/cmd/api_key_test.go @@ -0,0 +1,35 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: mpolden + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAPIKey(t *testing.T) { + t.Run("auth api-key", func(t *testing.T) { + testAPIKey(t, []string{"auth", "api-key"}) + }) +} + +func testAPIKey(t *testing.T, subcommand []string) { + cli, stdout, stderr := newTestCLI(t) + + err := cli.Run("config", "set", "target", "cloud") + assert.Nil(t, err) + + args := append(subcommand, "-a", "t1.a1.i1") + err = cli.Run(args...) + assert.Nil(t, err) + assert.Equal(t, "", stderr.String()) + assert.Contains(t, stdout.String(), "Success: API private key written to") + + err = cli.Run(subcommand...) + assert.NotNil(t, err) + assert.Contains(t, stderr.String(), "Error: refusing to overwrite") + assert.Contains(t, stderr.String(), "Hint: Use -f to overwrite it\n") + assert.Contains(t, stdout.String(), "This is your public key") +} diff --git a/client/go/internal/cli/cmd/auth.go b/client/go/internal/cli/cmd/auth.go new file mode 100644 index 00000000000..453d2296b08 --- /dev/null +++ b/client/go/internal/cli/cmd/auth.go @@ -0,0 +1,21 @@ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +func newAuthCmd() *cobra.Command { + return &cobra.Command{ + Use: "auth", + Short: "Manage Vespa Cloud credentials", + Long: `Manage Vespa Cloud credentials.`, + DisableAutoGenTag: true, + SilenceUsage: false, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return fmt.Errorf("invalid command: %s", args[0]) + }, + } +} diff --git a/client/go/internal/cli/cmd/cert.go b/client/go/internal/cli/cmd/cert.go new file mode 100644 index 00000000000..7f79a9db358 --- /dev/null +++ b/client/go/internal/cli/cmd/cert.go @@ -0,0 +1,219 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa cert command +// Author: mpolden +package cmd + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/fatih/color" + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func newCertCmd(cli *CLI) *cobra.Command { + var ( + noApplicationPackage bool + overwriteCertificate bool + ) + cmd := &cobra.Command{ + Use: "cert", + Short: "Create a new private key and self-signed certificate for data-plane access with Vespa Cloud", + Long: `Create a new private key and self-signed certificate for data-plane access with Vespa Cloud. + +The private key and certificate will be stored in the Vespa CLI home directory +(see 'vespa help config'). Other commands will then automatically load the +certificate as necessary. The certificate will be added to your application +package specified as an argument to this command (default '.'). + +It's possible to override the private key and certificate used through +environment variables. This can be useful in continuous integration systems. + +Example of setting the certificate and key in-line: + + export VESPA_CLI_DATA_PLANE_CERT="my cert" + export VESPA_CLI_DATA_PLANE_KEY="my private key" + +Example of loading certificate and key from custom paths: + + export VESPA_CLI_DATA_PLANE_CERT_FILE=/path/to/cert + export VESPA_CLI_DATA_PLANE_KEY_FILE=/path/to/key + +Note that when overriding key pair through environment variables, that key pair +will always be used for all applications. It's not possible to specify an +application-specific key. + +Read more in https://cloud.vespa.ai/en/security/guide`, + Example: `$ vespa auth cert -a my-tenant.my-app.my-instance +$ vespa auth cert -a my-tenant.my-app.my-instance path/to/application/package`, + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return doCert(cli, overwriteCertificate, noApplicationPackage, args) + }, + } + cmd.Flags().BoolVarP(&overwriteCertificate, "force", "f", false, "Force overwrite of existing certificate and private key") + cmd.Flags().BoolVarP(&noApplicationPackage, "no-add", "N", false, "Do not add certificate to the application package") + cmd.MarkPersistentFlagRequired(applicationFlag) + return cmd +} + +func newCertAddCmd(cli *CLI) *cobra.Command { + var overwriteCertificate bool + cmd := &cobra.Command{ + Use: "add", + Short: "Add certificate to application package", + Long: `Add an existing self-signed certificate for Vespa Cloud deployment to your application package. + +The certificate will be loaded from the Vespa CLI home directory (see 'vespa +help config') by default. + +The location of the application package can be specified as an argument to this +command (default '.').`, + Example: `$ vespa auth cert add -a my-tenant.my-app.my-instance +$ vespa auth cert add -a my-tenant.my-app.my-instance path/to/application/package`, + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return doCertAdd(cli, overwriteCertificate, args) + }, + } + cmd.Flags().BoolVarP(&overwriteCertificate, "force", "f", false, "Force overwrite of existing certificate") + cmd.MarkPersistentFlagRequired(applicationFlag) + return cmd +} + +func doCert(cli *CLI, overwriteCertificate, noApplicationPackage bool, args []string) error { + app, err := cli.config.application() + if err != nil { + return err + } + var pkg vespa.ApplicationPackage + if !noApplicationPackage { + pkg, err = cli.applicationPackageFrom(args, false) + if err != nil { + return err + } + } + targetType, err := cli.config.targetType() + if err != nil { + return err + } + privateKeyFile, err := cli.config.privateKeyPath(app, targetType) + if err != nil { + return err + } + certificateFile, err := cli.config.certificatePath(app, targetType) + if err != nil { + return err + } + + if !overwriteCertificate { + hint := "Use -f flag to force overwriting" + if !noApplicationPackage { + if pkg.HasCertificate() { + return errHint(fmt.Errorf("application package %s already contains a certificate", pkg.Path), hint) + } + } + if util.PathExists(privateKeyFile) { + return errHint(fmt.Errorf("private key %s already exists", color.CyanString(privateKeyFile)), hint) + } + if util.PathExists(certificateFile) { + return errHint(fmt.Errorf("certificate %s already exists", color.CyanString(certificateFile)), hint) + } + } + if !noApplicationPackage { + if pkg.IsZip() { + hint := "Try running 'mvn clean' before 'vespa auth cert', and then 'mvn package'" + return errHint(fmt.Errorf("cannot add certificate to compressed application package %s", pkg.Path), hint) + } + } + + keyPair, err := vespa.CreateKeyPair() + if err != nil { + return err + } + var pkgCertificateFile string + if !noApplicationPackage { + pkgCertificateFile = filepath.Join(pkg.Path, "security", "clients.pem") + if err := os.MkdirAll(filepath.Dir(pkgCertificateFile), 0755); err != nil { + return fmt.Errorf("could not create security directory: %w", err) + } + if err := keyPair.WriteCertificateFile(pkgCertificateFile, overwriteCertificate); err != nil { + return fmt.Errorf("could not write certificate to application package: %w", err) + } + } + if err := keyPair.WriteCertificateFile(certificateFile, overwriteCertificate); err != nil { + return fmt.Errorf("could not write certificate: %w", err) + } + if err := keyPair.WritePrivateKeyFile(privateKeyFile, overwriteCertificate); err != nil { + return fmt.Errorf("could not write private key: %w", err) + } + if !noApplicationPackage { + cli.printSuccess("Certificate written to ", color.CyanString(pkgCertificateFile)) + } + cli.printSuccess("Certificate written to ", color.CyanString(certificateFile)) + cli.printSuccess("Private key written to ", color.CyanString(privateKeyFile)) + return nil +} + +func doCertAdd(cli *CLI, overwriteCertificate bool, args []string) error { + app, err := cli.config.application() + if err != nil { + return err + } + pkg, err := cli.applicationPackageFrom(args, false) + if err != nil { + return err + } + targetType, err := cli.config.targetType() + if err != nil { + return err + } + certificateFile, err := cli.config.certificatePath(app, targetType) + if err != nil { + return err + } + + if pkg.IsZip() { + hint := "Try running 'mvn clean' before 'vespa auth cert add', and then 'mvn package'" + return errHint(fmt.Errorf("unable to add certificate to compressed application package: %s", pkg.Path), hint) + } + + pkgCertificateFile := filepath.Join(pkg.Path, "security", "clients.pem") + if err := os.MkdirAll(filepath.Dir(pkgCertificateFile), 0755); err != nil { + return fmt.Errorf("could not create security directory: %w", err) + } + src, err := os.Open(certificateFile) + if errors.Is(err, os.ErrNotExist) { + return errHint(fmt.Errorf("there is not key pair generated for application '%s'", app), "Try running 'vespa auth cert' to generate it") + } else if err != nil { + return fmt.Errorf("could not open certificate file: %w", err) + } + defer src.Close() + flags := os.O_CREATE | os.O_RDWR + if overwriteCertificate { + flags |= os.O_TRUNC + } else { + flags |= os.O_EXCL + } + dst, err := os.OpenFile(pkgCertificateFile, flags, 0755) + if errors.Is(err, os.ErrExist) { + return errHint(fmt.Errorf("application package %s already contains a certificate", pkg.Path), "Use -f flag to force overwriting") + } else if err != nil { + return fmt.Errorf("could not open application certificate file for writing: %w", err) + } + if _, err := io.Copy(dst, src); err != nil { + return fmt.Errorf("could not copy certificate file to application: %w", err) + } + + cli.printSuccess("Certificate written to ", color.CyanString(pkgCertificateFile)) + return nil +} diff --git a/client/go/internal/cli/cmd/cert_test.go b/client/go/internal/cli/cmd/cert_test.go new file mode 100644 index 00000000000..d6b47083e7b --- /dev/null +++ b/client/go/internal/cli/cmd/cert_test.go @@ -0,0 +1,129 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: mpolden + +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/mock" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func TestCert(t *testing.T) { + t.Run("auth cert", func(t *testing.T) { + testCert(t, []string{"auth", "cert"}) + }) +} + +func testCert(t *testing.T, subcommand []string) { + appDir, pkgDir := mock.ApplicationPackageDir(t, false, false) + + cli, stdout, stderr := newTestCLI(t) + args := append(subcommand, "-a", "t1.a1.i1", pkgDir) + err := cli.Run(args...) + assert.Nil(t, err) + + app, err := vespa.ApplicationFromString("t1.a1.i1") + assert.Nil(t, err) + + pkgCertificate := filepath.Join(appDir, "security", "clients.pem") + homeDir := cli.config.homeDir + certificate := filepath.Join(homeDir, app.String(), "data-plane-public-cert.pem") + privateKey := filepath.Join(homeDir, app.String(), "data-plane-private-key.pem") + + assert.Equal(t, fmt.Sprintf("Success: Certificate written to %s\nSuccess: Certificate written to %s\nSuccess: Private key written to %s\n", pkgCertificate, certificate, privateKey), stdout.String()) + + args = append(subcommand, "-a", "t1.a1.i1", pkgDir) + err = cli.Run(args...) + assert.NotNil(t, err) + assert.Contains(t, stderr.String(), fmt.Sprintf("Error: application package %s already contains a certificate", appDir)) +} + +func TestCertCompressedPackage(t *testing.T) { + t.Run("auth cert", func(t *testing.T) { + testCertCompressedPackage(t, []string{"auth", "cert"}) + }) +} + +func testCertCompressedPackage(t *testing.T, subcommand []string) { + _, pkgDir := mock.ApplicationPackageDir(t, true, false) + zipFile := filepath.Join(pkgDir, "target", "application.zip") + err := os.MkdirAll(filepath.Dir(zipFile), 0755) + assert.Nil(t, err) + _, err = os.Create(zipFile) + assert.Nil(t, err) + + cli, stdout, stderr := newTestCLI(t) + + args := append(subcommand, "-a", "t1.a1.i1", pkgDir) + err = cli.Run(args...) + assert.NotNil(t, err) + assert.Contains(t, stderr.String(), "Error: cannot add certificate to compressed application package") + + err = os.Remove(zipFile) + assert.Nil(t, err) + + args = append(subcommand, "-f", "-a", "t1.a1.i1", pkgDir) + err = cli.Run(args...) + assert.Nil(t, err) + assert.Contains(t, stdout.String(), "Success: Certificate written to") + assert.Contains(t, stdout.String(), "Success: Private key written to") +} + +func TestCertAdd(t *testing.T) { + cli, stdout, stderr := newTestCLI(t) + err := cli.Run("auth", "cert", "-N", "-a", "t1.a1.i1") + assert.Nil(t, err) + + appDir, pkgDir := mock.ApplicationPackageDir(t, false, false) + stdout.Reset() + err = cli.Run("auth", "cert", "add", "-a", "t1.a1.i1", pkgDir) + assert.Nil(t, err) + pkgCertificate := filepath.Join(appDir, "security", "clients.pem") + assert.Equal(t, fmt.Sprintf("Success: Certificate written to %s\n", pkgCertificate), stdout.String()) + + err = cli.Run("auth", "cert", "add", "-a", "t1.a1.i1", pkgDir) + assert.NotNil(t, err) + assert.Contains(t, stderr.String(), fmt.Sprintf("Error: application package %s already contains a certificate", appDir)) + stdout.Reset() + err = cli.Run("auth", "cert", "add", "-f", "-a", "t1.a1.i1", pkgDir) + assert.Nil(t, err) + assert.Equal(t, fmt.Sprintf("Success: Certificate written to %s\n", pkgCertificate), stdout.String()) +} + +func TestCertNoAdd(t *testing.T) { + cli, stdout, stderr := newTestCLI(t) + + err := cli.Run("auth", "cert", "-N", "-a", "t1.a1.i1") + assert.Nil(t, err) + homeDir := cli.config.homeDir + + app, err := vespa.ApplicationFromString("t1.a1.i1") + assert.Nil(t, err) + + certificate := filepath.Join(homeDir, app.String(), "data-plane-public-cert.pem") + privateKey := filepath.Join(homeDir, app.String(), "data-plane-private-key.pem") + assert.Equal(t, fmt.Sprintf("Success: Certificate written to %s\nSuccess: Private key written to %s\n", certificate, privateKey), stdout.String()) + + err = cli.Run("auth", "cert", "-N", "-a", "t1.a1.i1") + assert.NotNil(t, err) + assert.Contains(t, stderr.String(), fmt.Sprintf("Error: private key %s already exists", privateKey)) + require.Nil(t, os.Remove(privateKey)) + + stderr.Reset() + err = cli.Run("auth", "cert", "-N", "-a", "t1.a1.i1") + assert.NotNil(t, err) + assert.Contains(t, stderr.String(), fmt.Sprintf("Error: certificate %s already exists", certificate)) + + stdout.Reset() + err = cli.Run("auth", "cert", "-N", "-f", "-a", "t1.a1.i1") + assert.Nil(t, err) + assert.Equal(t, fmt.Sprintf("Success: Certificate written to %s\nSuccess: Private key written to %s\n", certificate, privateKey), stdout.String()) +} diff --git a/client/go/internal/cli/cmd/clone.go b/client/go/internal/cli/cmd/clone.go new file mode 100644 index 00000000000..6fb97581ea3 --- /dev/null +++ b/client/go/internal/cli/cmd/clone.go @@ -0,0 +1,288 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa clone command +// author: bratseth + +package cmd + +import ( + "archive/zip" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/fatih/color" + "github.com/spf13/cobra" +) + +const sampleAppsNamePrefix = "sample-apps-master" + +func newCloneCmd(cli *CLI) *cobra.Command { + var ( + listApps bool + noCache bool + ) + cmd := &cobra.Command{ + Use: "clone sample-application-path target-directory", + Short: "Create files and directory structure for a new Vespa application from a sample application", + Long: `Create files and directory structure for a new Vespa application +from a sample application. + +Sample applications are downloaded from +https://github.com/vespa-engine/sample-apps. + +By default sample applications are cached in the user's cache directory. This +directory can be overriden by setting the VESPA_CLI_CACHE_DIR environment +variable.`, + Example: "$ vespa clone album-recommendation my-app", + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + if listApps { + apps, err := listSampleApps(cli.httpClient) + if err != nil { + return fmt.Errorf("could not list sample applications: %w", err) + } + for _, app := range apps { + log.Print(app) + } + return nil + } + if len(args) != 2 { + return fmt.Errorf("expected exactly 2 arguments, got %d", len(args)) + } + cloner := &cloner{cli: cli, noCache: noCache} + return cloner.Clone(args[0], args[1]) + }, + } + cmd.Flags().BoolVarP(&listApps, "list", "l", false, "List available sample applications") + cmd.Flags().BoolVarP(&noCache, "force", "f", false, "Ignore cache and force downloading the latest sample application from GitHub") + return cmd +} + +type cloner struct { + cli *CLI + noCache bool +} + +type zipFile struct { + path string + etag string + modTime time.Time +} + +// Clone copies the application identified by applicationName into given path. If the cached copy of sample applications +// has expired (as determined by its entity tag), a current copy will be downloaded from GitHub automatically. +func (c *cloner) Clone(applicationName, path string) error { + zipPath, err := c.zipPath() + if err != nil { + return err + } + + r, err := zip.OpenReader(zipPath) + if err != nil { + return fmt.Errorf("could not open sample apps zip '%s': %w", color.CyanString(zipPath), err) + } + defer r.Close() + + found := false + for _, f := range r.File { + dirPrefix := "sample-apps-master/" + applicationName + "/" + if strings.HasPrefix(f.Name, dirPrefix) { + if !found { // Create destination directory lazily when source is found + createErr := os.Mkdir(path, 0755) + if createErr != nil { + return fmt.Errorf("could not create directory '%s': %w", color.CyanString(path), createErr) + } + } + found = true + + if err := copyFromZip(f, path, dirPrefix); err != nil { + return fmt.Errorf("could not copy zip entry '%s': %w", color.CyanString(f.Name), err) + } + } + } + + if !found { + return errHint(fmt.Errorf("could not find source application '%s'", color.CyanString(applicationName)), "Use -f to ignore the cache") + } else { + log.Print("Created ", color.CyanString(path)) + } + return nil +} + +// zipPath returns the path to the latest sample application ZIP file. +func (c *cloner) zipPath() (string, error) { + zipFiles, err := c.listZipFiles() + if err != nil { + return "", nil + } + cacheCandidates := zipFiles + if c.noCache { + cacheCandidates = nil + } + zipPath, cacheHit, err := c.downloadZip(cacheCandidates) + if err != nil { + if cacheHit { + c.cli.printWarning(err) + } else { + return "", err + } + } + if cacheHit { + log.Print(color.YellowString("Using cached sample apps ...")) + } + // Remove obsolete files + for _, zf := range zipFiles { + if zf.path != zipPath { + os.Remove(zf.path) + } + } + return zipPath, nil +} + +// listZipFiles list all sample apps ZIP files found in cacheDir. +func (c *cloner) listZipFiles() ([]zipFile, error) { + dirEntries, err := os.ReadDir(c.cli.config.cacheDir) + if err != nil { + return nil, err + } + var zipFiles []zipFile + for _, entry := range dirEntries { + ext := filepath.Ext(entry.Name()) + if ext != ".zip" { + continue + } + if !strings.HasPrefix(entry.Name(), sampleAppsNamePrefix) { + continue + } + fi, err := entry.Info() + if err != nil { + return nil, err + } + name := fi.Name() + etag := "" + parts := strings.Split(name, "_") + if len(parts) == 2 { + etag = strings.TrimSuffix(parts[1], ext) + } + zipFiles = append(zipFiles, zipFile{ + path: filepath.Join(c.cli.config.cacheDir, name), + etag: etag, + modTime: fi.ModTime(), + }) + } + return zipFiles, nil +} + +// downloadZip conditionally downloads the latest sample apps ZIP file. If any of the ZIP files among cacheFiles are +// usable, downloading is skipped. +func (c *cloner) downloadZip(cachedFiles []zipFile) (string, bool, error) { + zipPath := "" + etag := "" + sort.Slice(cachedFiles, func(i, j int) bool { return cachedFiles[i].modTime.Before(cachedFiles[j].modTime) }) + if len(cachedFiles) > 0 { + latest := cachedFiles[len(cachedFiles)-1] + zipPath = latest.path + etag = latest.etag + } + // The latest cached file, if any, is considered a hit until we have downloaded a fresh one. This allows us to use + // the cached copy if GitHub is unavailable. + cacheHit := zipPath != "" + err := c.cli.spinner(c.cli.Stderr, color.YellowString("Downloading sample apps ..."), func() error { + request, err := http.NewRequest("GET", "https://github.com/vespa-engine/sample-apps/archive/refs/heads/master.zip", nil) + if err != nil { + return fmt.Errorf("invalid url: %w", err) + } + if etag != "" { + request.Header = make(http.Header) + request.Header.Set("if-none-match", fmt.Sprintf(`W/"%s"`, etag)) + } + response, err := c.cli.httpClient.Do(request, time.Minute*60) + if err != nil { + return fmt.Errorf("could not download sample apps: %w", err) + } + defer response.Body.Close() + if response.StatusCode == http.StatusNotModified { // entity tag matched so our cached copy is current + return nil + } + if response.StatusCode != http.StatusOK { + return fmt.Errorf("could not download sample apps: github returned status %d", response.StatusCode) + } + etag = trimEntityTagID(response.Header.Get("etag")) + newPath, err := c.writeZip(response.Body, etag) + if err != nil { + return err + } + zipPath = newPath + cacheHit = false + return nil + }) + return zipPath, cacheHit, err +} + +// writeZip atomically writes the contents of reader zipReader to a file in the CLI cache directory. +func (c *cloner) writeZip(zipReader io.Reader, etag string) (string, error) { + f, err := os.CreateTemp(c.cli.config.cacheDir, "sample-apps-tmp-") + if err != nil { + return "", fmt.Errorf("could not create temporary file: %w", err) + } + cleanTemp := true + defer func() { + f.Close() + if cleanTemp { + os.Remove(f.Name()) + } + }() + if _, err := io.Copy(f, zipReader); err != nil { + return "", fmt.Errorf("could not write sample apps to file: %s: %w", f.Name(), err) + } + f.Close() + path := filepath.Join(c.cli.config.cacheDir, sampleAppsNamePrefix) + if etag != "" { + path += "_" + etag + } + path += ".zip" + if err := os.Rename(f.Name(), path); err != nil { + return "", fmt.Errorf("could not move sample apps to %s", path) + } + cleanTemp = false + return path, nil +} + +func trimEntityTagID(s string) string { + return strings.TrimSuffix(strings.TrimPrefix(s, `W/"`), `"`) +} + +func copyFromZip(f *zip.File, destinationDir string, zipEntryPrefix string) error { + destinationPath := filepath.Join(destinationDir, filepath.FromSlash(strings.TrimPrefix(f.Name, zipEntryPrefix))) + if strings.HasSuffix(f.Name, "/") { + if f.Name != zipEntryPrefix { // root is already created + if err := os.Mkdir(destinationPath, 0755); err != nil { + return err + } + } + } else { + r, err := f.Open() + if err != nil { + return err + } + defer r.Close() + destination, err := os.Create(destinationPath) + if err != nil { + return err + } + if _, err := io.Copy(destination, r); err != nil { + return err + } + if err := os.Chmod(destinationPath, f.Mode()); err != nil { + return err + } + } + return nil +} diff --git a/client/go/internal/cli/cmd/clone_list.go b/client/go/internal/cli/cmd/clone_list.go new file mode 100644 index 00000000000..826aea7b75e --- /dev/null +++ b/client/go/internal/cli/cmd/clone_list.go @@ -0,0 +1,86 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package cmd + +import ( + "encoding/json" + "net/http" + "sort" + "time" + + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +func listSampleApps(client util.HTTPClient) ([]string, error) { + return listSampleAppsAt("https://api.github.com/repos/vespa-engine/sample-apps/contents/", client) +} + +func listSampleAppsAt(url string, client util.HTTPClient) ([]string, error) { + rfs, err := getRepositoryFiles(url, client) + if err != nil { + return nil, err + } + var apps []string + for _, rf := range rfs { + isApp, follow := isApp(rf) + if isApp { + apps = append(apps, rf.Path) + } else if follow { + apps2, err := listSampleAppsAt(rf.URL, client) + if err != nil { + return nil, err + } + apps = append(apps, apps2...) + } + } + sort.Strings(apps) + return apps, nil +} + +func getRepositoryFiles(url string, client util.HTTPClient) ([]repositoryFile, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + response, err := client.Do(req, time.Minute) + if err != nil { + return nil, err + } + defer response.Body.Close() + var files []repositoryFile + dec := json.NewDecoder(response.Body) + if err := dec.Decode(&files); err != nil { + return nil, err + } + return files, nil +} + +func isApp(rf repositoryFile) (ok bool, follow bool) { + if rf.Type != "dir" { + return false, false + } + if rf.Path == "" { + return false, false + } + if rf.Path[0] == '_' || rf.Path[0] == '.' { + return false, false + } + // These are just heuristics and must be updated if we add more directories that are not applications, or that + // contain multiple applications inside + switch rf.Name { + case "test", "bin", "src": + return false, false + } + switch rf.Path { + case "news", "examples", "examples/operations", "operations", "vespa-cloud": + return false, true + } + return true, false +} + +type repositoryFile struct { + Path string `json:"path"` + Name string `json:"name"` + Type string `json:"type"` + URL string `json:"url"` + HtmlURL string `json:"html_url"` +} diff --git a/client/go/internal/cli/cmd/clone_list_test.go b/client/go/internal/cli/cmd/clone_list_test.go new file mode 100644 index 00000000000..f69ad2be8cf --- /dev/null +++ b/client/go/internal/cli/cmd/clone_list_test.go @@ -0,0 +1,67 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/mock" +) + +func TestListSampleApps(t *testing.T) { + c := &mock.HTTPClient{} + c.NextResponseString(200, readTestData(t, "sample-apps-contents.json")) + c.NextResponseString(200, readTestData(t, "sample-apps-news.json")) + c.NextResponseString(200, readTestData(t, "sample-apps-operations.json")) + c.NextResponseString(200, readTestData(t, "sample-apps-vespa-cloud.json")) + + apps, err := listSampleApps(c) + assert.Nil(t, err) + expected := []string{ + "album-recommendation-monitoring", + "album-recommendation-selfhosted", + "basic-search-on-gke", + "boolean-search", + "dense-passage-retrieval-with-ann", + "generic-request-processing", + "http-api-using-request-handlers-and-processors", + "incremental-search", + "model-evaluation", + "msmarco-ranking", + "multiple-bundles", + "multiple-bundles-lib", + "news/app-1-getting-started", + "news/app-2-feed-and-query", + "news/app-3-searching", + "news/app-5-recommendation", + "news/app-6-recommendation-with-searchers", + "news/app-7-parent-child", + "operations/multinode", + "part-purchases-demo", + "secure-vespa-with-mtls", + "semantic-qa-retrieval", + "tensor-playground", + "text-search", + "transformers", + "use-case-shopping", + "vespa-chinese-linguistics", + "vespa-cloud/album-recommendation", + "vespa-cloud/album-recommendation-docproc", + "vespa-cloud/album-recommendation-prod", + "vespa-cloud/album-recommendation-searcher", + "vespa-cloud/cord-19-search", + "vespa-cloud/joins", + "vespa-cloud/vespa-documentation-search", + } + assert.Equal(t, expected, apps) +} + +func readTestData(t *testing.T, name string) string { + contents, err := os.ReadFile(filepath.Join("testdata", name)) + if err != nil { + t.Fatal(err) + } + return string(contents) +} diff --git a/client/go/internal/cli/cmd/clone_test.go b/client/go/internal/cli/cmd/clone_test.go new file mode 100644 index 00000000000..f136299db85 --- /dev/null +++ b/client/go/internal/cli/cmd/clone_test.go @@ -0,0 +1,94 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// init command tests +// Author: bratseth + +package cmd + +import ( + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vespa-engine/vespa/client/go/internal/mock" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +func TestClone(t *testing.T) { + assertCreated("text-search", "mytestapp", t) +} + +func assertCreated(sampleAppName string, app string, t *testing.T) { + tempDir := t.TempDir() + app1 := filepath.Join(tempDir, "app1") + defer os.RemoveAll(app) + + httpClient := &mock.HTTPClient{} + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = httpClient + testdata, err := os.ReadFile(filepath.Join("testdata", "sample-apps-master.zip")) + require.Nil(t, err) + + // Initial cloning. GitHub includes the ETag header, but we don't require it + httpClient.NextResponseBytes(200, testdata) + require.Nil(t, cli.Run("clone", sampleAppName, app1)) + assert.Equal(t, "Created "+app1+"\n", stdout.String()) + assertFiles(t, app1) + + // Clone with cache hit + httpClient.NextStatus(http.StatusNotModified) + stdout.Reset() + app2 := filepath.Join(tempDir, "app2") + require.Nil(t, cli.Run("clone", sampleAppName, app2)) + assert.Equal(t, "Using cached sample apps ...\nCreated "+app2+"\n", stdout.String()) + assertFiles(t, app2) + + // Clone while ignoring cache + headers := make(http.Header) + headers.Set("etag", `W/"id1"`) + httpClient.NextResponse(mock.HTTPResponse{Status: 200, Body: testdata, Header: headers}) + stdout.Reset() + app3 := filepath.Join(tempDir, "app3") + require.Nil(t, cli.Run("clone", "-f", sampleAppName, app3)) + assert.Equal(t, "Created "+app3+"\n", stdout.String()) + assertFiles(t, app3) + + // Cloning falls back to cached copy if GitHub is unavailable + httpClient.NextStatus(500) + stdout.Reset() + app4 := filepath.Join(tempDir, "app4") + require.Nil(t, cli.Run("clone", "-f=false", sampleAppName, app4)) + assert.Equal(t, "Warning: could not download sample apps: github returned status 500\n", stderr.String()) + assert.Equal(t, "Using cached sample apps ...\nCreated "+app4+"\n", stdout.String()) + assertFiles(t, app4) + + // The only cached file is the latest one + dirEntries, err := os.ReadDir(cli.config.cacheDir) + require.Nil(t, err) + var zipFiles []string + for _, de := range dirEntries { + name := de.Name() + if strings.HasPrefix(name, sampleAppsNamePrefix) { + zipFiles = append(zipFiles, name) + } + } + assert.Equal(t, []string{"sample-apps-master_id1.zip"}, zipFiles) +} + +func assertFiles(t *testing.T, app string) { + assert.True(t, util.PathExists(filepath.Join(app, "README.md"))) + assert.True(t, util.PathExists(filepath.Join(app, "src", "main", "application"))) + assert.True(t, util.IsDirectory(filepath.Join(app, "src", "main", "application"))) + + servicesStat, err := os.Stat(filepath.Join(app, "src", "main", "application", "services.xml")) + require.Nil(t, err) + servicesSize := int64(1772) + assert.Equal(t, servicesSize, servicesStat.Size()) + + scriptStat, err := os.Stat(filepath.Join(app, "bin", "convert-msmarco.sh")) + require.Nil(t, err) + assert.Equal(t, os.FileMode(0755), scriptStat.Mode()) +} diff --git a/client/go/internal/cli/cmd/clusterstate/cluster_state.go b/client/go/internal/cli/cmd/clusterstate/cluster_state.go new file mode 100644 index 00000000000..7317e9a8a3a --- /dev/null +++ b/client/go/internal/cli/cmd/clusterstate/cluster_state.go @@ -0,0 +1,122 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +// utilities to get and manipulate node states in a storage cluster +package clusterstate + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +// common struct used various places in the clustercontroller REST api: +type StateAndReason struct { + State string `json:"state"` + Reason string `json:"reason"` +} + +func (s *StateAndReason) writeTo(buf *strings.Builder) { + buf.WriteString(s.State) + if s.Reason != "" { + buf.WriteString(" [reason: ") + buf.WriteString(s.Reason) + buf.WriteString("]") + } +} + +// cluster state as returned by the clustercontroller REST api: +type ClusterState struct { + State struct { + Generated StateAndReason `json:"generated"` + } `json:"state"` + Service map[string]struct { + Node map[string]struct { + Attributes struct { + HierarchicalGroup string `json:"hierarchical-group"` + } `json:"attributes"` + State struct { + Generated StateAndReason `json:"generated"` + Unit StateAndReason `json:"unit"` + User StateAndReason `json:"user"` + } `json:"state"` + Metrics struct { + BucketCount int `json:"bucket-count"` + UniqueDocumentCount int `json:"unique-document-count"` + UniqueDocumentTotalSize int `json:"unique-document-total-size"` + } `json:"metrics"` + } `json:"node"` + } `json:"service"` + DistributionStates struct { + Published struct { + Baseline string `json:"baseline"` + BucketSpaces []struct { + Name string `json:"name"` + State string `json:"state"` + } `json:"bucket-spaces"` + } `json:"published"` + } `json:"distribution-states"` +} + +func (cs *ClusterState) String() string { + if cs == nil { + return "nil" + } + var buf strings.Builder + buf.WriteString("cluster state: ") + cs.State.Generated.writeTo(&buf) + for n, s := range cs.Service { + buf.WriteString("\n ") + buf.WriteString(n) + buf.WriteString(": [") + for nn, node := range s.Node { + buf.WriteString("\n ") + buf.WriteString(nn) + buf.WriteString(" -> {generated: ") + node.State.Generated.writeTo(&buf) + buf.WriteString("} {unit: ") + node.State.Unit.writeTo(&buf) + buf.WriteString("} {user: ") + node.State.User.writeTo(&buf) + buf.WriteString("}") + } + } + buf.WriteString("\n") + return buf.String() +} + +func (model *VespaModelConfig) getClusterState(cluster string) (*ClusterState, *ClusterControllerSpec) { + errs := make([]string, 0, 0) + ccs := model.findClusterControllers() + if len(ccs) == 0 { + trace.Trace("No cluster controllers found in vespa model:", model) + errs = append(errs, "No cluster controllers found in vespa model config") + } + for _, cc := range ccs { + url := fmt.Sprintf("http://%s:%d/cluster/v2/%s/?recursive=true", + cc.host, cc.port, cluster) + var buf bytes.Buffer + err := curlGet(url, &buf) + if err != nil { + errs = append(errs, "could not get: "+url) + continue + } + codec := json.NewDecoder(&buf) + var parsedJson ClusterState + err = codec.Decode(&parsedJson) + if err != nil { + trace.Trace("Could not parse JSON >>>", buf.String(), "<<< from", url) + errs = append(errs, "Bad JSON from "+url+" was: "+buf.String()) + continue + } + // success: + return &parsedJson, &cc + } + // no success: + util.JustExitMsg(fmt.Sprint(errs)) + panic("unreachable") +} diff --git a/client/go/internal/cli/cmd/clusterstate/detect_model.go b/client/go/internal/cli/cmd/clusterstate/detect_model.go new file mode 100644 index 00000000000..bb1192d4106 --- /dev/null +++ b/client/go/internal/cli/cmd/clusterstate/detect_model.go @@ -0,0 +1,67 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +// utilities to get and manipulate node states in a storage cluster +package clusterstate + +import ( + "strconv" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func getConfigServerHosts(s string) []string { + if s != "" { + return []string{s} + } + backticks := util.BackTicksForwardStderr + got, err := backticks.Run(vespa.FindHome()+"/bin/vespa-print-default", "configservers") + res := strings.Fields(got) + if err != nil || len(res) < 1 { + util.JustExitMsg("bad configservers: " + got) + } + trace.Debug("found", len(res), "configservers:", res) + return res +} + +func getConfigServerPort(i int) int { + if i > 0 { + return i + } + backticks := util.BackTicksForwardStderr + got, err := backticks.Run(vespa.FindHome()+"/bin/vespa-print-default", "configserver_rpc_port") + if err == nil { + i, err = strconv.Atoi(strings.TrimSpace(got)) + } + if err != nil || i < 1 { + util.JustExitMsg("bad configserver_rpc_port: " + got) + } + trace.Debug("found configservers rpc port:", i) + return i +} + +func detectModel(opts *Options) *VespaModelConfig { + vespa.LoadDefaultEnv() + cfgHosts := getConfigServerHosts(opts.ConfigServerHost) + cfgPort := getConfigServerPort(opts.ConfigServerPort) + for _, cfgHost := range cfgHosts { + args := []string{ + "-j", + "-n", "cloud.config.model", + "-i", "admin/model", + "-p", strconv.Itoa(cfgPort), + "-s", cfgHost, + } + backticks := util.BackTicksForwardStderr + data, err := backticks.Run(vespa.FindHome()+"/bin/vespa-get-config", args...) + parsed := parseModelConfig(data) + if err == nil && parsed != nil { + return parsed + } + } + util.JustExitMsg("could not get model config") + panic("unreachable") +} diff --git a/client/go/internal/cli/cmd/clusterstate/get_cluster_state.go b/client/go/internal/cli/cmd/clusterstate/get_cluster_state.go new file mode 100644 index 00000000000..505235a284e --- /dev/null +++ b/client/go/internal/cli/cmd/clusterstate/get_cluster_state.go @@ -0,0 +1,77 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// code for the "vespa-get-cluster-state" command +// Author: arnej + +// utilities to get and manipulate node states in a storage cluster +package clusterstate + +import ( + "fmt" + "os" + + "github.com/fatih/color" + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/cli/build" +) + +func NewGetClusterStateCmd() *cobra.Command { + var ( + curOptions Options + ) + cmd := &cobra.Command{ + Use: "vespa-get-cluster-state [-h] [-v] [-f] [-c cluster]", + Short: "Get the cluster state of a given cluster.", + Long: `Usage: get-cluster-state [Options]`, + Version: build.Version, + Args: cobra.MaximumNArgs(0), + CompletionOptions: cobra.CompletionOptions{DisableDefaultCmd: true}, + Run: func(cmd *cobra.Command, args []string) { + curOptions.NodeIndex = AllNodes + runGetClusterState(&curOptions) + }, + } + addCommonOptions(cmd, &curOptions) + return cmd +} + +func runGetClusterState(opts *Options) { + if opts.Silent { + trace.Silent() + } + if opts.NoColors || os.Getenv(envvars.TERM) == "" { + color.NoColor = true + } + trace.Debug("run getClusterState with: ", opts) + m := detectModel(opts) + trace.Debug("model:", m) + sss := m.findSelectedServices(opts) + clusters := make(map[string]*ClusterState) + for _, s := range sss { + trace.Debug("found service: ", s) + if clusters[s.cluster] == nil { + state, _ := m.getClusterState(s.cluster) + trace.Debug("cluster ", s.cluster, state) + clusters[s.cluster] = state + } + } + for k, v := range clusters { + globalState := v.State.Generated.State + if globalState == "up" { + fmt.Printf("Cluster %s:\n", k) + } else { + fmt.Printf("Cluster %s is %s. Too few nodes available.\n", k, color.HiRedString("%s", globalState)) + } + for serviceType, serviceList := range v.Service { + for dn, dv := range serviceList.Node { + nodeState := dv.State.Generated.State + if nodeState == "up" { + fmt.Printf("%s/%s/%s: %v\n", k, serviceType, dn, nodeState) + } else { + fmt.Printf("%s/%s/%s: %v\n", k, serviceType, dn, color.HiRedString(nodeState)) + } + } + } + } +} diff --git a/client/go/internal/cli/cmd/clusterstate/get_node_state.go b/client/go/internal/cli/cmd/clusterstate/get_node_state.go new file mode 100644 index 00000000000..6d45e377a72 --- /dev/null +++ b/client/go/internal/cli/cmd/clusterstate/get_node_state.go @@ -0,0 +1,100 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// code for the "vespa-get-node-state" command +// Author: arnej + +// utilities to get and manipulate node states in a storage cluster +package clusterstate + +import ( + "fmt" + "os" + "strconv" + + "github.com/fatih/color" + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/cli/build" +) + +const ( + longdesc = `Retrieve the state of one or more storage services from the fleet controller. Will list the state of the locally running services, possibly restricted to less by options.` + header = `Shows the various states of one or more nodes in a Vespa Storage cluster. There exist three different type of node states. They are: + + Unit state - The state of the node seen from the cluster controller. + User state - The state we want the node to be in. By default up. Can be + set by administrators or by cluster controller when it + detects nodes that are behaving badly. + Generated state - The state of a given node in the current cluster state. + This is the state all the other nodes know about. This + state is a product of the other two states and cluster + controller logic to keep the cluster stable.` +) + +func NewGetNodeStateCmd() *cobra.Command { + var ( + curOptions Options + ) + cmd := &cobra.Command{ + Use: "vespa-get-node-state [-h] [-v] [-c cluster] [-t type] [-i index]", + Short: "Get the state of a node.", + Long: longdesc + "\n\n" + header, + Version: build.Version, + Args: cobra.MaximumNArgs(0), + CompletionOptions: cobra.CompletionOptions{DisableDefaultCmd: true}, + Run: func(cmd *cobra.Command, args []string) { + runGetNodeState(&curOptions) + }, + } + addCommonOptions(cmd, &curOptions) + cmd.Flags().StringVarP(&curOptions.NodeType, "type", "t", "", + "Node type - can either be 'storage' or 'distributor'. If not specified, the operation will use state for both types.") + cmd.Flags().IntVarP(&curOptions.NodeIndex, "index", "i", OnlyLocalNode, + "Node index. If not specified, all nodes found running on this host will be used.") + return cmd +} + +func runGetNodeState(opts *Options) { + if opts.Silent { + trace.Silent() + } + if opts.NoColors || os.Getenv(envvars.TERM) == "" { + color.NoColor = true + } + trace.Info(header) + m := detectModel(opts) + sss := m.findSelectedServices(opts) + clusters := make(map[string]*ClusterState) + for _, s := range sss { + state := clusters[s.cluster] + if state == nil { + state, _ = m.getClusterState(s.cluster) + clusters[s.cluster] = state + } + if state == nil { + trace.Warning("no state for cluster: ", s.cluster) + continue + } + if nodes, ok := state.Service[s.serviceType]; ok { + for name, node := range nodes.Node { + if name == strconv.Itoa(s.index) { + fmt.Printf("\n%s/%s.%s:\n", s.cluster, s.serviceType, name) + dumpState(node.State.Unit, "Unit") + dumpState(node.State.Generated, "Generated") + dumpState(node.State.User, "User") + } + } + } else { + trace.Warning("no nodes for service type: ", s.serviceType) + continue + } + + } +} + +func dumpState(s StateAndReason, tag string) { + if s.State != "up" { + s.State = color.HiRedString(s.State) + } + fmt.Printf("%s: %s: %s\n", tag, s.State, s.Reason) +} diff --git a/client/go/internal/cli/cmd/clusterstate/known_state.go b/client/go/internal/cli/cmd/clusterstate/known_state.go new file mode 100644 index 00000000000..60a4ada7711 --- /dev/null +++ b/client/go/internal/cli/cmd/clusterstate/known_state.go @@ -0,0 +1,35 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +// utilities to get and manipulate node states in a storage cluster +package clusterstate + +import ( + "fmt" +) + +type KnownState string + +// these are all the valid node states: +const ( + StateUp KnownState = "up" + StateDown KnownState = "down" + StateMaintenance KnownState = "maintenance" + StateRetired KnownState = "retired" +) + +// verify that a string is one of the known states: +func knownState(s string) (KnownState, error) { + alternatives := []KnownState{ + StateUp, + StateDown, + StateMaintenance, + StateRetired, + } + for _, v := range alternatives { + if s == string(v) { + return v, nil + } + } + return KnownState("unknown"), fmt.Errorf("<Wanted State> must be one of %v, was %s\n", alternatives, s) +} diff --git a/client/go/internal/cli/cmd/clusterstate/model_config.go b/client/go/internal/cli/cmd/clusterstate/model_config.go new file mode 100644 index 00000000000..5d0e9d98200 --- /dev/null +++ b/client/go/internal/cli/cmd/clusterstate/model_config.go @@ -0,0 +1,126 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +// utilities to get and manipulate node states in a storage cluster +package clusterstate + +import ( + "encoding/json" + "sort" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" +) + +type VespaModelConfig struct { + VespaVersion string `json:"vespaVersion"` + Hosts []struct { + Name string `json:"name"` + Services []struct { + Name string `json:"name"` + Type string `json:"type"` + Configid string `json:"configid"` + Clustertype string `json:"clustertype"` + Clustername string `json:"clustername"` + Index int `json:"index"` + Ports []struct { + Number int `json:"number"` + Tags string `json:"tags"` + } `json:"ports"` + } `json:"services"` + } `json:"hosts"` +} + +func (m *VespaModelConfig) String() string { + if m == nil { + return "nil" + } + var buf strings.Builder + buf.WriteString("vespa version: ") + buf.WriteString(m.VespaVersion) + for _, h := range m.Hosts { + buf.WriteString("\n host: ") + buf.WriteString(h.Name) + for _, s := range h.Services { + buf.WriteString("\n service: ") + buf.WriteString(s.Name) + buf.WriteString(" type: ") + buf.WriteString(s.Type) + buf.WriteString(" cluster: ") + buf.WriteString(s.Clustername) + } + buf.WriteString("\n") + } + buf.WriteString("\n") + return buf.String() +} + +type ClusterControllerSpec struct { + host string + port int +} + +func parseModelConfig(input string) *VespaModelConfig { + codec := json.NewDecoder(strings.NewReader(input)) + var parsedJson VespaModelConfig + err := codec.Decode(&parsedJson) + if err != nil { + trace.Trace("could not decode JSON >>>", input, "<<< error:", err) + return nil + } + return &parsedJson +} + +func (m *VespaModelConfig) findClusterControllers() []ClusterControllerSpec { + res := make([]ClusterControllerSpec, 0, 1) + for _, h := range m.Hosts { + for _, s := range h.Services { + if s.Type == "container-clustercontroller" { + for _, p := range s.Ports { + if strings.Contains(p.Tags, "state") { + res = append(res, ClusterControllerSpec{ + host: h.Name, port: p.Number, + }) + } + } + } + } + } + return res +} + +func (m *VespaModelConfig) findSelectedServices(opts *Options) []serviceSpec { + res := make([]serviceSpec, 0, 5) + for _, h := range m.Hosts { + for _, s := range h.Services { + spec := serviceSpec{ + cluster: s.Clustername, + serviceType: s.Type, + index: s.Index, + host: h.Name, + } + if s.Type == "storagenode" { + // simplify: + spec.serviceType = "storage" + } + if opts.wantService(spec) { + res = append(res, spec) + } + } + } + sort.Slice(res, func(i, j int) bool { + a := res[i] + b := res[j] + if a.cluster != b.cluster { + return a.cluster < b.cluster + } + if a.serviceType != b.serviceType { + return a.serviceType < b.serviceType + } + if a.index != b.index { + return a.index < b.index + } + return a.host < b.host + }) + return res +} diff --git a/client/go/internal/cli/cmd/clusterstate/options.go b/client/go/internal/cli/cmd/clusterstate/options.go new file mode 100644 index 00000000000..b58562f9abe --- /dev/null +++ b/client/go/internal/cli/cmd/clusterstate/options.go @@ -0,0 +1,149 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +// utilities to get and manipulate node states in a storage cluster +package clusterstate + +import ( + "strconv" + "strings" + + "github.com/fatih/color" + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +const ( + OnlyLocalNode int = -2 + AllNodes int = -1 +) + +type Options struct { + Verbose int + Silent bool + ShowHidden showHiddenFlag + Force bool + NoColors bool + SafeMode bool + NoWait bool + Cluster string + ConfigServerHost string + ConfigServerPort int + ConfigRequestTimeout int + NodeType string + NodeIndex int + WantedState string +} + +func (v *Options) String() string { + var buf strings.Builder + buf.WriteString("command-line options [") + if v.Verbose > 0 { + buf.WriteString(" verbosity=") + buf.WriteString(strconv.Itoa(v.Verbose)) + } + if v.Silent { + buf.WriteString(" silent") + } + if v.ShowHidden.showHidden { + buf.WriteString(" show-hidden") + } + if v.Force { + buf.WriteString(color.HiYellowString(" force=true")) + } + if v.NoColors { + buf.WriteString(" no-colors") + } + if v.SafeMode { + buf.WriteString(" safe-mode") + } + if v.NoWait { + buf.WriteString(color.HiYellowString(" no-wait=true")) + } + if v.Cluster != "" { + buf.WriteString(" cluster=") + buf.WriteString(v.Cluster) + } + if v.ConfigServerHost != "" { + buf.WriteString(" config-server=") + buf.WriteString(v.ConfigServerHost) + } + if v.ConfigServerPort != 0 { + buf.WriteString(" config-server-port=") + buf.WriteString(strconv.Itoa(v.ConfigServerPort)) + } + if v.ConfigRequestTimeout != 90 { + buf.WriteString(" config-request-timeout=") + buf.WriteString(strconv.Itoa(v.ConfigRequestTimeout)) + } + if v.NodeType != "" { + buf.WriteString(" node-type=") + buf.WriteString(v.NodeType) + } + if v.NodeIndex >= 0 { + buf.WriteString(" node-index=") + buf.WriteString(strconv.Itoa(int(v.NodeIndex))) + } + if v.WantedState != "" { + buf.WriteString(" WantedState=") + buf.WriteString(v.WantedState) + } + buf.WriteString(" ]") + return buf.String() +} + +type serviceSpec struct { + cluster string + serviceType string + index int + host string +} + +func (o *Options) wantService(s serviceSpec) bool { + if o.Cluster != "" && o.Cluster != s.cluster { + return false + } + if o.NodeType == "" { + if s.serviceType != "storage" && s.serviceType != "distributor" { + return false + } + } else if o.NodeType != s.serviceType { + return false + } + switch o.NodeIndex { + case OnlyLocalNode: + myName, _ := vespa.FindOurHostname() + return s.host == "localhost" || s.host == myName + case AllNodes: + return true + case s.index: + return true + default: + return false + } +} + +func addCommonOptions(cmd *cobra.Command, curOptions *Options) { + cmd.Flags().BoolVar(&curOptions.NoColors, "nocolors", false, "Do not use ansi colors in print.") + cmd.Flags().BoolVarP(&curOptions.Silent, "silent", "s", false, "Create less verbose output.") + cmd.Flags().CountVarP(&curOptions.Verbose, "verbose", "v", "Create more verbose output.") + cmd.Flags().IntVar(&curOptions.ConfigRequestTimeout, "config-request-timeout", 90, "Timeout of config request") + cmd.Flags().IntVar(&curOptions.ConfigServerPort, "config-server-port", 0, "Port to connect to config server on") + cmd.Flags().StringVar(&curOptions.ConfigServerHost, "config-server", "", "Host name of config server to query") + cmd.Flags().StringVarP(&curOptions.Cluster, "cluster", "c", "", + "Cluster name. If unspecified, and vespa is installed on current node, information will be attempted auto-extracted") + cmd.Flags().MarkHidden("config-request-timeout") + cmd.Flags().MarkHidden("config-server-port") + cmd.Flags().MarkHidden("nocolors") + curOptions.ShowHidden.cmd = cmd + flag := cmd.Flags().VarPF(&curOptions.ShowHidden, "show-hidden", "", "Also show hidden undocumented debug options.") + flag.NoOptDefVal = "true" + cobra.OnInitialize(func() { + if curOptions.Silent { + trace.Silent() + } else { + trace.AdjustVerbosity(curOptions.Verbose) + } + }) +} diff --git a/client/go/internal/cli/cmd/clusterstate/run_curl.go b/client/go/internal/cli/cmd/clusterstate/run_curl.go new file mode 100644 index 00000000000..1dcb31528e1 --- /dev/null +++ b/client/go/internal/cli/cmd/clusterstate/run_curl.go @@ -0,0 +1,85 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +// utilities to get and manipulate node states in a storage cluster +package clusterstate + +import ( + "bytes" + "fmt" + "io" + "os" + "os/exec" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/curl" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func curlCommand(url string, args []string) (*curl.Command, error) { + tls, err := vespa.LoadTlsConfig() + if err != nil { + return nil, err + } + if tls != nil && strings.HasPrefix(url, "http:") { + url = "https:" + url[5:] + } + cmd, err := curl.RawArgs(url, args...) + if err != nil { + return nil, err + } + if tls != nil { + if tls.DisableHostnameValidation { + cmd, err = curl.RawArgs(url, append(args, "--insecure")...) + if err != nil { + return nil, err + } + } + cmd.PrivateKey = tls.Files.PrivateKey + cmd.Certificate = tls.Files.Certificates + cmd.CaCertificate = tls.Files.CaCertificates + } + return cmd, err +} + +func curlGet(url string, output io.Writer) error { + cmd, err := curlCommand(url, commonCurlArgs()) + if err != nil { + return err + } + trace.Trace("running curl:", cmd.String()) + err = cmd.Run(output, os.Stderr) + return err +} + +func curlPost(url string, input []byte) (string, error) { + cmd, err := curlCommand(url, commonCurlArgs()) + cmd.Method = "POST" + cmd.Header("Content-Type", "application/json") + cmd.WithBodyInput(bytes.NewReader(input)) + var out bytes.Buffer + trace.Debug("POST input: " + string(input)) + trace.Trace("running curl:", cmd.String()) + err = cmd.Run(&out, os.Stderr) + if err != nil { + if ee, ok := err.(*exec.ExitError); ok { + if ee.ProcessState.ExitCode() == 7 { + return "", fmt.Errorf("HTTP request to %s failed, could not connect", url) + } + } + return "", fmt.Errorf("HTTP request failed with curl %s", err.Error()) + } + return out.String(), err +} + +func commonCurlArgs() []string { + return []string{ + "-A", "vespa-cluster-state", + "--silent", + "--show-error", + "--connect-timeout", "30", + "--max-time", "1200", + "--write-out", "\n%{http_code}", + } +} diff --git a/client/go/internal/cli/cmd/clusterstate/set_node_state.go b/client/go/internal/cli/cmd/clusterstate/set_node_state.go new file mode 100644 index 00000000000..2a6869c84f5 --- /dev/null +++ b/client/go/internal/cli/cmd/clusterstate/set_node_state.go @@ -0,0 +1,155 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// code for the "vespa-set-node-state" command +// Author: arnej + +// utilities to get and manipulate node states in a storage cluster +package clusterstate + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + + "github.com/fatih/color" + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/cli/build" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +const ( + usageSetNodeState = `vespa-set-node-state [Options] <Wanted State> [Description] + +Arguments: + Wanted State : User state to set. This must be one of up, down, maintenance or retired. + Description : Give a reason for why you are altering the user state, which will show up in various admin tools. (Use double quotes to give a reason + with whitespace in it)` + + longSetNodeState = `Set the user state of a node. This will set the generated state to the user state if the user state is "better" than the generated state that would +have been created if the user state was up. For instance, a node that is currently in initializing state can be forced into down state, while a node +that is currently down can not be forced into retired state, but can be forced into maintenance state.` +) + +func NewSetNodeStateCmd() *cobra.Command { + var ( + curOptions Options + ) + cmd := &cobra.Command{ + Use: usageSetNodeState, + Short: "vespa-set-node-state [Options] <Wanted State> [Description]", + Long: longSetNodeState, + Version: build.Version, + Args: func(cmd *cobra.Command, args []string) error { + switch { + case len(args) < 1: + return fmt.Errorf("Missing <Wanted State>") + case len(args) > 2: + return fmt.Errorf("Too many arguments, maximum is 2") + } + _, err := knownState(args[0]) + return err + }, + CompletionOptions: cobra.CompletionOptions{DisableDefaultCmd: true}, + Run: func(cmd *cobra.Command, args []string) { + runSetNodeState(&curOptions, args) + }, + } + addCommonOptions(cmd, &curOptions) + cmd.Flags().BoolVarP(&curOptions.Force, "force", "f", false, + "Force execution") + cmd.Flags().BoolVarP(&curOptions.NoWait, "no-wait", "n", false, + "Do not wait for node state changes to be visible in the cluster before returning.") + cmd.Flags().BoolVarP(&curOptions.SafeMode, "safe", "a", false, + "Only carries out state changes if deemed safe by the cluster controller.") + cmd.Flags().StringVarP(&curOptions.NodeType, "type", "t", "", + "Node type - can either be 'storage' or 'distributor'. If not specified, the operation will set state for both types.") + cmd.Flags().IntVarP(&curOptions.NodeIndex, "index", "i", OnlyLocalNode, + "Node index. If not specified, all nodes found running on this host will be used.") + cmd.Flags().MarkHidden("no-wait") + return cmd +} + +func runSetNodeState(opts *Options, args []string) { + if opts.Silent { + trace.Silent() + } + if opts.NoColors || os.Getenv(envvars.TERM) == "" { + color.NoColor = true + } + wanted, err := knownState(args[0]) + if err != nil { + util.JustExitWith(err) + } + reason := "" + if len(args) > 1 { + reason = args[1] + } + if !opts.Force && wanted == StateMaintenance && opts.NodeType != "storage" { + fmt.Println(color.HiYellowString( + `Setting the distributor to maintenance mode may have severe consequences for feeding! +Please specify -t storage to only set the storage node to maintenance mode, or -f to override this error.`)) + return + } + m := detectModel(opts) + sss := m.findSelectedServices(opts) + if len(sss) == 0 { + fmt.Println(color.HiYellowString("Attempted setting of user state for no nodes")) + return + } + for _, s := range sss { + _, cc := m.getClusterState(s.cluster) + cc.setNodeUserState(s, wanted, reason, opts) + } +} + +type SetNodeStateJson struct { + State struct { + User StateAndReason `json:"user"` + } `json:"state"` + ResponseWait string `json:"response-wait,omitempty"` + Condition string `json:"condition,omitempty"` +} + +func splitResultCode(s string) (int, string) { + for idx := len(s); idx > 0; { + idx-- + if s[idx] == '\n' { + resCode, err := strconv.Atoi(s[idx+1:]) + if err != nil { + return -1, s + } + return resCode, s[:idx] + } + } + return -1, s +} + +func (cc *ClusterControllerSpec) setNodeUserState(s serviceSpec, wanted KnownState, reason string, opts *Options) error { + var request SetNodeStateJson + request.State.User.State = string(wanted) + request.State.User.Reason = reason + if opts.NoWait { + request.ResponseWait = "no-wait" + } + if opts.SafeMode { + request.Condition = "safe" + } + jsonBytes, err := json.Marshal(request) + if err != nil { + util.JustExitWith(err) + } + url := fmt.Sprintf("http://%s:%d/cluster/v2/%s/%s/%d", + cc.host, cc.port, + s.cluster, s.serviceType, s.index) + result, err := curlPost(url, jsonBytes) + resCode, output := splitResultCode(result) + if resCode < 200 || resCode >= 300 { + fmt.Println(color.HiYellowString("failed with HTTP code %d", resCode)) + fmt.Println(output) + } else { + fmt.Print(output, "OK\n") + } + return err +} diff --git a/client/go/internal/cli/cmd/clusterstate/show_hidden.go b/client/go/internal/cli/cmd/clusterstate/show_hidden.go new file mode 100644 index 00000000000..8c0ef61bf18 --- /dev/null +++ b/client/go/internal/cli/cmd/clusterstate/show_hidden.go @@ -0,0 +1,36 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +// utilities to get and manipulate node states in a storage cluster +package clusterstate + +import ( + "strconv" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +// handle CLI flag --show-hidden + +type showHiddenFlag struct { + showHidden bool + cmd *cobra.Command +} + +func (v *showHiddenFlag) Type() string { + return "" +} + +func (v *showHiddenFlag) String() string { + return strconv.FormatBool(v.showHidden) +} + +func (v *showHiddenFlag) Set(val string) error { + b, err := strconv.ParseBool(val) + v.showHidden = b + v.cmd.Flags().VisitAll(func(f *pflag.Flag) { f.Hidden = false }) + return err +} + +func (v *showHiddenFlag) IsBoolFlag() bool { return true } diff --git a/client/go/internal/cli/cmd/config.go b/client/go/internal/cli/cmd/config.go new file mode 100644 index 00000000000..e5124962831 --- /dev/null +++ b/client/go/internal/cli/cmd/config.go @@ -0,0 +1,697 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa config command +// author: bratseth + +package cmd + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "log" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "github.com/fatih/color" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/vespa-engine/vespa/client/go/internal/cli/auth/auth0" + "github.com/vespa-engine/vespa/client/go/internal/cli/config" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +const ( + configFile = "config.yaml" +) + +func newConfigCmd() *cobra.Command { + return &cobra.Command{ + Use: "config", + Short: "Configure persistent values for global flags", + Long: `Configure persistent values for global flags. + +This command allows setting persistent values for global flags. On future +invocations the flag can then be omitted as it is read from the config file +instead. + +Configuration is written to $HOME/.vespa by default. This path can be +overridden by setting the VESPA_CLI_HOME environment variable. + +When setting an option locally, the configuration is written to .vespa in the +working directory, where that directory is assumed to be a Vespa application +directory. This allows you have separate configuration options per application. + +Vespa CLI chooses the value for a given option in the following order, from +most to least preferred: + +1. Flag value specified on the command line +2. Local config value +3. Global config value +4. Default value + +The following flags/options can be configured: + +application + +Specifies the application ID to manage. It has three parts, separated by +dots, with the third part being optional. This is only relevant for the "cloud" +and "hosted" targets. See https://cloud.vespa.ai/en/tenant-apps-instances for +more details. This has no default value. Examples: tenant1.app1, +tenant1.app1.instance1 + +cluster + +Specifies the container cluster to manage. If left empty (default) and the +application has only one container cluster, that cluster is chosen +automatically. When an application has multiple cluster this must be set a +valid cluster name, as specified in services.xml. See +https://docs.vespa.ai/en/reference/services-container.html for more details. + +color + +Controls how Vespa CLI uses colors. Setting this to "auto" (default) enables +colors if supported by the terminal, "never" completely disables colors and +"always" enables colors unilaterally. + +instance + +Specifies the instance of the application to manage. When specified, this takes +precedence over the instance specified as part of application. This has no +default value. Example: instance2 + +quiet + +Print only errors. + +target + +Specifies the target to use for commands that interact with a Vespa platform, +e.g. vespa deploy or vespa query. Possible values are: + +- local: (default) Connect to a Vespa platform running at localhost +- cloud: Connect to Vespa Cloud +- hosted: Connect to hosted Vespa (internal platform) +- *url*: Connect to a platform running at given URL. + +wait + +Specifies the number of seconds to wait for a service to become ready or +deployment to complete. Use this to have a potentially long-running command +block until the operation is complete, e.g. with vespa deploy. Defaults to 0 +(no waiting) + +zone + +Specifies a custom dev or perf zone to use when connecting to a Vespa platform. +This is only relevant for cloud and hosted targets. By default, a zone is +chosen automatically. See https://cloud.vespa.ai/en/reference/zones for +available zones. Examples: dev.aws-us-east-1c, perf.aws-us-east-1c +`, + DisableAutoGenTag: true, + SilenceUsage: false, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return fmt.Errorf("invalid command: %s", args[0]) + }, + } +} + +func newConfigSetCmd(cli *CLI) *cobra.Command { + var localArg bool + cmd := &cobra.Command{ + Use: "set option-name value", + Short: "Set a configuration option.", + Example: `# Set the target to Vespa Cloud +$ vespa config set target cloud + +# Set application, without a specific instance. The instance will be named "default" +$ vespa config set application my-tenant.my-application + +# Set application with a specific instance +$ vespa config set application my-tenant.my-application.my-instance + +# Set the instance explicitly. This will take precedence over an instance specified as part of the application option. +$ vespa config set instance other-instance + +# Set an option in local configuration, for the current application only +$ vespa config set --local wait 600 +`, + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + config := cli.config + if localArg { + // Need an application package in working directory to allow local configuration + if _, err := cli.applicationPackageFrom(nil, false); err != nil { + return fmt.Errorf("failed to write local configuration: %w", err) + } + config = cli.config.local + } + if err := config.set(args[0], args[1]); err != nil { + return err + } + return config.write() + }, + } + cmd.Flags().BoolVarP(&localArg, "local", "l", false, "Write option to local configuration, i.e. for the current application") + return cmd +} + +func newConfigUnsetCmd(cli *CLI) *cobra.Command { + var localArg bool + cmd := &cobra.Command{ + Use: "unset option-name", + Short: "Unset a configuration option.", + Long: `Unset a configuration option. + +Unsetting a configuration option will reset it to its default value, which may be empty. +`, + Example: `# Reset target to its default value +$ vespa config unset target + +# Stop overriding application option in local config +$ vespa config unset --local application +`, + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + config := cli.config + if localArg { + if _, err := cli.applicationPackageFrom(nil, false); err != nil { + return fmt.Errorf("failed to write local configuration: %w", err) + } + config = cli.config.local + } + if err := config.unset(args[0]); err != nil { + return err + } + return config.write() + }, + } + cmd.Flags().BoolVarP(&localArg, "local", "l", false, "Unset option in local configuration, i.e. for the current application") + return cmd +} + +func newConfigGetCmd(cli *CLI) *cobra.Command { + var localArg bool + cmd := &cobra.Command{ + Use: "get [option-name]", + Short: "Show given configuration option, or all configuration options", + Long: `Show given configuration option, or all configuration options. + +By default this command prints the effective configuration for the current +application, i.e. it takes into account any local configuration located in +[working-directory]/.vespa. +`, + Example: `$ vespa config get +$ vespa config get target +$ vespa config get --local +`, + Args: cobra.MaximumNArgs(1), + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + config := cli.config + if localArg { + if cli.config.local.isEmpty() { + cli.printWarning("no local configuration present") + return nil + } + config = cli.config.local + } + if len(args) == 0 { // Print all values + for _, option := range config.list(!localArg) { + config.printOption(option) + } + } else { + return config.printOption(args[0]) + } + return nil + }, + } + cmd.Flags().BoolVarP(&localArg, "local", "l", false, "Show only local configuration, if any") + return cmd +} + +type Config struct { + homeDir string + cacheDir string + environment map[string]string + local *Config + + flags map[string]*pflag.Flag + config *config.Config +} + +type KeyPair struct { + KeyPair tls.Certificate + CertificateFile string + PrivateKeyFile string +} + +func loadConfig(environment map[string]string, flags map[string]*pflag.Flag) (*Config, error) { + home, err := vespaCliHome(environment) + if err != nil { + return nil, fmt.Errorf("could not detect config directory: %w", err) + } + config, err := loadConfigFrom(home, environment, flags) + if err != nil { + return nil, err + } + // Load local config from working directory by default + if err := config.loadLocalConfigFrom("."); err != nil { + return nil, err + } + return config, nil +} + +func loadConfigFrom(dir string, environment map[string]string, flags map[string]*pflag.Flag) (*Config, error) { + cacheDir, err := vespaCliCacheDir(environment) + if err != nil { + return nil, fmt.Errorf("could not detect cache directory: %w", err) + } + c := &Config{ + homeDir: dir, + cacheDir: cacheDir, + environment: environment, + flags: flags, + } + f, err := os.Open(filepath.Join(dir, configFile)) + var cfg *config.Config + if os.IsNotExist(err) { + cfg = config.New() + } else if err != nil { + return nil, err + } else { + defer f.Close() + cfg, err = config.Read(f) + if err != nil { + return nil, err + } + } + c.config = cfg + return c, nil +} + +func athenzPath(filename string) (string, error) { + userHome, err := os.UserHomeDir() + if err != nil { + return "", err + } + return filepath.Join(userHome, ".athenz", filename), nil +} + +func (c *Config) loadLocalConfigFrom(parent string) error { + home := filepath.Join(parent, ".vespa") + _, err := os.Stat(home) + if err != nil && !os.IsNotExist(err) { + return err + } + config, err := loadConfigFrom(home, c.environment, c.flags) + if err != nil { + return err + } + c.local = config + return nil +} + +func (c *Config) write() error { + if err := os.MkdirAll(c.homeDir, 0700); err != nil { + return err + } + configFile := filepath.Join(c.homeDir, configFile) + return c.config.WriteFile(configFile) +} + +func (c *Config) targetType() (string, error) { + targetType, ok := c.get(targetFlag) + if !ok { + return "", fmt.Errorf("target is unset") + } + return targetType, nil +} + +func (c *Config) timeout() (time.Duration, error) { + wait, ok := c.get(waitFlag) + if !ok { + return 0, nil + } + secs, err := strconv.Atoi(wait) + if err != nil { + return 0, err + } + return time.Duration(secs) * time.Second, nil +} + +func (c *Config) isQuiet() bool { + quiet, _ := c.get(quietFlag) + return quiet == "true" +} + +func (c *Config) application() (vespa.ApplicationID, error) { + app, ok := c.get(applicationFlag) + if !ok { + return vespa.ApplicationID{}, errHint(fmt.Errorf("no application specified"), "Try the --"+applicationFlag+" flag") + } + application, err := vespa.ApplicationFromString(app) + if err != nil { + return vespa.ApplicationID{}, errHint(err, "application format is <tenant>.<app>[.<instance>]") + } + instance, ok := c.get(instanceFlag) + if ok { + application.Instance = instance + } + return application, nil +} + +func (c *Config) cluster() string { + cluster, _ := c.get(clusterFlag) + return cluster +} + +func (c *Config) deploymentIn(system vespa.System) (vespa.Deployment, error) { + zone := system.DefaultZone + zoneName, ok := c.get(zoneFlag) + if ok { + var err error + zone, err = vespa.ZoneFromString(zoneName) + if err != nil { + return vespa.Deployment{}, err + } + } + app, err := c.application() + if err != nil { + return vespa.Deployment{}, err + } + return vespa.Deployment{System: system, Application: app, Zone: zone}, nil +} + +func (c *Config) certificatePath(app vespa.ApplicationID, targetType string) (string, error) { + if override, ok := c.environment["VESPA_CLI_DATA_PLANE_CERT_FILE"]; ok { + return override, nil + } + if targetType == vespa.TargetHosted { + return athenzPath("cert") + } + return c.applicationFilePath(app, "data-plane-public-cert.pem") +} + +func (c *Config) privateKeyPath(app vespa.ApplicationID, targetType string) (string, error) { + if override, ok := c.environment["VESPA_CLI_DATA_PLANE_KEY_FILE"]; ok { + return override, nil + } + if targetType == vespa.TargetHosted { + return athenzPath("key") + } + return c.applicationFilePath(app, "data-plane-private-key.pem") +} + +func (c *Config) x509KeyPair(app vespa.ApplicationID, targetType string) (KeyPair, error) { + cert, certOk := c.environment["VESPA_CLI_DATA_PLANE_CERT"] + key, keyOk := c.environment["VESPA_CLI_DATA_PLANE_KEY"] + var ( + kp tls.Certificate + err error + certFile string + keyFile string + ) + if certOk && keyOk { + // Use key pair from environment + kp, err = tls.X509KeyPair([]byte(cert), []byte(key)) + } else { + keyFile, err = c.privateKeyPath(app, targetType) + if err != nil { + return KeyPair{}, err + } + certFile, err = c.certificatePath(app, targetType) + if err != nil { + return KeyPair{}, err + } + kp, err = tls.LoadX509KeyPair(certFile, keyFile) + } + if err != nil { + return KeyPair{}, err + } + if targetType == vespa.TargetHosted { + cert, err := x509.ParseCertificate(kp.Certificate[0]) + if err != nil { + return KeyPair{}, err + } + now := time.Now() + expiredAt := cert.NotAfter + if expiredAt.Before(now) { + delta := now.Sub(expiredAt).Truncate(time.Second) + return KeyPair{}, fmt.Errorf("certificate %s expired at %s (%s ago)", certFile, cert.NotAfter, delta) + } + return KeyPair{KeyPair: kp, CertificateFile: certFile, PrivateKeyFile: keyFile}, nil + } + return KeyPair{ + KeyPair: kp, + CertificateFile: certFile, + PrivateKeyFile: keyFile, + }, nil +} + +func (c *Config) apiKeyFileFromEnv() (string, bool) { + override, ok := c.environment["VESPA_CLI_API_KEY_FILE"] + return override, ok +} + +func (c *Config) apiKeyFromEnv() ([]byte, bool) { + override, ok := c.environment["VESPA_CLI_API_KEY"] + return []byte(override), ok +} + +func (c *Config) apiKeyPath(tenantName string) string { + if override, ok := c.apiKeyFileFromEnv(); ok { + return override + } + return filepath.Join(c.homeDir, tenantName+".api-key.pem") +} + +func (c *Config) authConfigPath() string { + return filepath.Join(c.homeDir, "auth.json") +} + +func (c *Config) readAPIKey(cli *CLI, system vespa.System, tenantName string) ([]byte, error) { + if override, ok := c.apiKeyFromEnv(); ok { + return override, nil + } + if path, ok := c.apiKeyFileFromEnv(); ok { + return os.ReadFile(path) + } + if cli.isCloudCI() { + return nil, nil // Vespa Cloud CI only talks to data plane and does not have an API key + } + if !cli.isCI() { + client, err := auth0.New(c.authConfigPath(), system.Name, system.URL) + if err == nil && client.HasCredentials() { + return nil, nil // use Auth0 + } + cli.printWarning("Authenticating with API key. This is discouraged in non-CI environments", "Authenticate with 'vespa auth login'") + } + return os.ReadFile(c.apiKeyPath(tenantName)) +} + +func (c *Config) readSessionID(app vespa.ApplicationID) (int64, error) { + sessionPath, err := c.applicationFilePath(app, "session_id") + if err != nil { + return 0, err + } + b, err := os.ReadFile(sessionPath) + if err != nil { + return 0, err + } + return strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64) +} + +func (c *Config) writeSessionID(app vespa.ApplicationID, sessionID int64) error { + sessionPath, err := c.applicationFilePath(app, "session_id") + if err != nil { + return err + } + return os.WriteFile(sessionPath, []byte(fmt.Sprintf("%d\n", sessionID)), 0600) +} + +func (c *Config) applicationFilePath(app vespa.ApplicationID, name string) (string, error) { + appDir := filepath.Join(c.homeDir, app.String()) + if err := os.MkdirAll(appDir, 0700); err != nil { + return "", err + } + return filepath.Join(appDir, name), nil +} + +func (c *Config) isEmpty() bool { return len(c.config.Keys()) == 0 } + +// list returns the options that have been set in this configuration. If includeUnset is true, also return options that +// haven't been set. +func (c *Config) list(includeUnset bool) []string { + if !includeUnset { + return c.config.Keys() + } + var flags []string + for k := range c.flags { + flags = append(flags, k) + } + sort.Strings(flags) + return flags +} + +// flagValue returns the set value and default value of the named flag. +func (c *Config) flagValue(name string) (string, string) { + f, ok := c.flags[name] + if !ok { + return "", "" + } + return f.Value.String(), f.DefValue +} + +// getNonEmpty returns value of given option, if that value is non-empty +func (c *Config) getNonEmpty(option string) (string, bool) { + v, ok := c.config.Get(option) + if v == "" { + return "", false + } + return v, ok +} + +// get returns the value associated with option, from the most preferred source in the following order: flag > local +// config > global config. +func (c *Config) get(option string) (string, bool) { + flagValue, flagDefault := c.flagValue(option) + // explicit flag value always takes precedence over everything else + if flagValue != flagDefault { + return flagValue, true + } + // ... then local config, if option is explicitly defined there + if c.local != nil { + if value, ok := c.local.getNonEmpty(option); ok { + return value, ok + } + } + // ... then global config + if v, ok := c.getNonEmpty(option); ok { + return v, ok + } + // ... then finally default flag value, if any + return flagDefault, flagDefault != "" +} + +func (c *Config) set(option, value string) error { + switch option { + case targetFlag: + switch value { + case vespa.TargetLocal, vespa.TargetCloud, vespa.TargetHosted: + c.config.Set(option, value) + return nil + } + if strings.HasPrefix(value, "http://") || strings.HasPrefix(value, "https://") { + c.config.Set(option, value) + return nil + } + case applicationFlag: + app, err := vespa.ApplicationFromString(value) + if err != nil { + return err + } + c.config.Set(option, app.String()) + return nil + case instanceFlag: + c.config.Set(option, value) + return nil + case clusterFlag: + c.config.Set(clusterFlag, value) + return nil + case waitFlag: + if n, err := strconv.Atoi(value); err != nil || n < 0 { + return fmt.Errorf("%s option must be an integer >= 0, got %q", option, value) + } + c.config.Set(option, value) + return nil + case colorFlag: + switch value { + case "auto", "never", "always": + c.config.Set(option, value) + return nil + } + case quietFlag: + switch value { + case "true", "false": + c.config.Set(option, value) + return nil + } + case zoneFlag: + if _, err := vespa.ZoneFromString(value); err != nil { + return err + } + c.config.Set(option, value) + return nil + } + return fmt.Errorf("invalid option or value: %s = %s", option, value) +} + +func (c *Config) unset(option string) error { + if err := c.checkOption(option); err != nil { + return err + } + c.config.Del(option) + return nil +} + +func (c *Config) checkOption(option string) error { + if _, ok := c.flags[option]; !ok { + return fmt.Errorf("invalid option: %s", option) + } + return nil +} + +func (c *Config) printOption(option string) error { + if err := c.checkOption(option); err != nil { + return err + } + value, ok := c.get(option) + if !ok { + faintColor := color.New(color.FgWhite, color.Faint) + value = faintColor.Sprint("<unset>") + } else { + value = color.CyanString(value) + } + log.Printf("%s = %s", option, value) + return nil +} + +func vespaCliHome(env map[string]string) (string, error) { + home := env["VESPA_CLI_HOME"] + if home == "" { + userHome, err := os.UserHomeDir() + if err != nil { + return "", err + } + home = filepath.Join(userHome, ".vespa") + } + if err := os.MkdirAll(home, 0700); err != nil { + return "", err + } + return home, nil +} + +func vespaCliCacheDir(env map[string]string) (string, error) { + cacheDir := env["VESPA_CLI_CACHE_DIR"] + if cacheDir == "" { + userCacheDir, err := os.UserCacheDir() + if err != nil { + return "", err + } + cacheDir = filepath.Join(userCacheDir, "vespa") + } + if err := os.MkdirAll(cacheDir, 0755); err != nil { + return "", err + } + return cacheDir, nil +} diff --git a/client/go/internal/cli/cmd/config_test.go b/client/go/internal/cli/cmd/config_test.go new file mode 100644 index 00000000000..c76277119a0 --- /dev/null +++ b/client/go/internal/cli/cmd/config_test.go @@ -0,0 +1,208 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vespa-engine/vespa/client/go/internal/mock" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func TestConfig(t *testing.T) { + configHome := t.TempDir() + assertConfigCommandErr(t, configHome, "Error: invalid option or value: foo = bar\n", "config", "set", "foo", "bar") + assertConfigCommandErr(t, configHome, "Error: invalid option: foo\n", "config", "get", "foo") + + // target + assertConfigCommand(t, configHome, "target = local\n", "config", "get", "target") // default value + assertConfigCommand(t, configHome, "", "config", "set", "target", "hosted") + assertConfigCommand(t, configHome, "target = hosted\n", "config", "get", "target") + assertConfigCommand(t, configHome, "", "config", "set", "target", "cloud") + assertConfigCommand(t, configHome, "target = cloud\n", "config", "get", "target") + assertConfigCommand(t, configHome, "", "config", "set", "target", "http://127.0.0.1:8080") + assertConfigCommand(t, configHome, "", "config", "set", "target", "https://127.0.0.1") + assertConfigCommand(t, configHome, "target = https://127.0.0.1\n", "config", "get", "target") + + // application + assertConfigCommandErr(t, configHome, "Error: invalid application: \"foo\"\n", "config", "set", "application", "foo") + assertConfigCommand(t, configHome, "application = <unset>\n", "config", "get", "application") + assertConfigCommand(t, configHome, "", "config", "set", "application", "t1.a1.i1") + assertConfigCommand(t, configHome, "application = t1.a1.i1\n", "config", "get", "application") + assertConfigCommand(t, configHome, "", "config", "set", "application", "t1.a1") + assertConfigCommand(t, configHome, "application = t1.a1.default\n", "config", "get", "application") + + // cluster + assertConfigCommand(t, configHome, "cluster = <unset>\n", "config", "get", "cluster") + assertConfigCommand(t, configHome, "", "config", "set", "cluster", "feed") + assertConfigCommand(t, configHome, "cluster = feed\n", "config", "get", "cluster") + + // instance + assertConfigCommand(t, configHome, "instance = <unset>\n", "config", "get", "instance") + assertConfigCommand(t, configHome, "", "config", "set", "instance", "i2") + assertConfigCommand(t, configHome, "instance = i2\n", "config", "get", "instance") + + // wait + assertConfigCommandErr(t, configHome, "Error: wait option must be an integer >= 0, got \"foo\"\n", "config", "set", "wait", "foo") + assertConfigCommand(t, configHome, "", "config", "set", "wait", "60") + assertConfigCommand(t, configHome, "wait = 60\n", "config", "get", "wait") + assertConfigCommand(t, configHome, "wait = 30\n", "config", "get", "--wait", "30", "wait") // flag overrides global config + + // color + assertConfigCommandErr(t, configHome, "Error: invalid option or value: color = foo\n", "config", "set", "color", "foo") + assertConfigCommand(t, configHome, "", "config", "set", "color", "never") + assertConfigCommand(t, configHome, "color = never\n", "config", "get", "color") + assertConfigCommand(t, configHome, "", "config", "unset", "color") + assertConfigCommand(t, configHome, "color = auto\n", "config", "get", "color") + + // quiet + assertConfigCommand(t, configHome, "", "config", "set", "quiet", "true") + assertConfigCommand(t, configHome, "", "config", "set", "quiet", "false") + + // zone + assertConfigCommand(t, configHome, "", "config", "set", "zone", "dev.us-east-1") + assertConfigCommand(t, configHome, "zone = dev.us-east-1\n", "config", "get", "zone") + + // Write empty value to YAML config, which should be ignored. This is for compatibility with older config formats + configFile := filepath.Join(configHome, "config.yaml") + assertConfigCommand(t, configHome, "", "config", "unset", "zone") + data, err := os.ReadFile(configFile) + require.Nil(t, err) + yamlConfig := string(data) + assert.NotContains(t, yamlConfig, "zone:") + config := yamlConfig + "zone: \"\"\n" + require.Nil(t, os.WriteFile(configFile, []byte(config), 0600)) + assertConfigCommand(t, configHome, "zone = <unset>\n", "config", "get", "zone") +} + +func TestLocalConfig(t *testing.T) { + configHome := t.TempDir() + // Write a few global options + assertConfigCommand(t, configHome, "", "config", "set", "instance", "main") + assertConfigCommand(t, configHome, "", "config", "set", "target", "cloud") + + // Change directory to an application package and write local options + _, rootDir := mock.ApplicationPackageDir(t, false, false) + wd, err := os.Getwd() + require.Nil(t, err) + t.Cleanup(func() { os.Chdir(wd) }) + require.Nil(t, os.Chdir(rootDir)) + assertConfigCommandStdErr(t, configHome, "Warning: no local configuration present\n", "config", "get", "--local") + assertConfigCommand(t, configHome, "", "config", "set", "--local", "instance", "foo") + assertConfigCommand(t, configHome, "instance = foo\n", "config", "get", "instance") + assertConfigCommand(t, configHome, "instance = bar\n", "config", "get", "--instance", "bar", "instance") // flag overrides local config + + // get --local prints only options set in local config + assertConfigCommand(t, configHome, "instance = foo\n", "config", "get", "--local") + + // get reads global option if unset locally + assertConfigCommand(t, configHome, "target = cloud\n", "config", "get", "target") + + // get merges settings from local and global config + assertConfigCommand(t, configHome, "", "config", "set", "--local", "application", "t1.a1") + assertConfigCommand(t, configHome, `application = t1.a1.default +cluster = <unset> +color = auto +instance = foo +quiet = false +target = cloud +wait = 0 +zone = <unset> +`, "config", "get") + + // Only locally set options are written + localConfig, err := os.ReadFile(filepath.Join(rootDir, ".vespa", "config.yaml")) + require.Nil(t, err) + assert.Equal(t, "application: t1.a1.default\ninstance: foo\n", string(localConfig)) + + // Changing back to original directory reads from global config + require.Nil(t, os.Chdir(wd)) + assertConfigCommand(t, configHome, "instance = main\n", "config", "get", "instance") + assertConfigCommand(t, configHome, "target = cloud\n", "config", "get", "target") +} + +func assertConfigCommand(t *testing.T, configHome, expected string, args ...string) { + t.Helper() + assertEnvConfigCommand(t, configHome, expected, nil, args...) +} + +func assertEnvConfigCommand(t *testing.T, configHome, expected string, env []string, args ...string) { + t.Helper() + env = append(env, "VESPA_CLI_HOME="+configHome) + cli, stdout, _ := newTestCLI(t, env...) + err := cli.Run(args...) + assert.Nil(t, err) + assert.Equal(t, expected, stdout.String()) +} + +func assertConfigCommandStdErr(t *testing.T, configHome, expected string, args ...string) error { + t.Helper() + cli, _, stderr := newTestCLI(t) + err := cli.Run(args...) + assert.Equal(t, expected, stderr.String()) + return err +} + +func assertConfigCommandErr(t *testing.T, configHome, expected string, args ...string) { + t.Helper() + assert.NotNil(t, assertConfigCommandStdErr(t, configHome, expected, args...)) +} + +func TestReadAPIKey(t *testing.T) { + cli, _, _ := newTestCLI(t) + key, err := cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + assert.Nil(t, key) + require.NotNil(t, err) + + // From default path when it exists + require.Nil(t, os.WriteFile(filepath.Join(cli.config.homeDir, "t1.api-key.pem"), []byte("foo"), 0600)) + key, err = cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + require.Nil(t, err) + assert.Equal(t, []byte("foo"), key) + + // Cloud CI does not read key from disk as it's not expected to have any + cli, _, _ = newTestCLI(t, "VESPA_CLI_CLOUD_CI=true") + key, err = cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + require.Nil(t, err) + assert.Nil(t, key) + + // From file specified in environment + keyFile := filepath.Join(t.TempDir(), "key") + require.Nil(t, os.WriteFile(keyFile, []byte("bar"), 0600)) + cli, _, _ = newTestCLI(t, "VESPA_CLI_API_KEY_FILE="+keyFile) + key, err = cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + require.Nil(t, err) + assert.Equal(t, []byte("bar"), key) + + // From key specified in environment + cli, _, _ = newTestCLI(t, "VESPA_CLI_API_KEY=baz") + key, err = cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + require.Nil(t, err) + assert.Equal(t, []byte("baz"), key) + + // Auth0 is preferred when configured + authContent := ` +{ + "version": 1, + "providers": { + "auth0": { + "version": 1, + "systems": { + "public": { + "access_token": "...", + "scopes": ["openid", "offline_access"], + "expires_at": "2030-01-01T01:01:01.000001+01:00" + } + } + } + } +}` + cli, _, _ = newTestCLI(t) + require.Nil(t, os.WriteFile(filepath.Join(cli.config.homeDir, "auth.json"), []byte(authContent), 0600)) + key, err = cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + require.Nil(t, err) + assert.Nil(t, key) +} diff --git a/client/go/internal/cli/cmd/curl.go b/client/go/internal/cli/cmd/curl.go new file mode 100644 index 00000000000..8fcd1fa6ef7 --- /dev/null +++ b/client/go/internal/cli/cmd/curl.go @@ -0,0 +1,98 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package cmd + +import ( + "fmt" + "log" + "net/http" + "os" + "strings" + + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/curl" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func newCurlCmd(cli *CLI) *cobra.Command { + var ( + dryRun bool + curlService string + ) + cmd := &cobra.Command{ + Use: "curl [curl-options] path", + Short: "Access Vespa directly using curl", + Long: `Access Vespa directly using curl. + +Execute curl with the appropriate URL, certificate and private key for your application. + +For a more high-level interface to query and feeding, see the 'query' and 'document' commands. +`, + Example: `$ vespa curl /ApplicationStatus +$ vespa curl -- -X POST -H "Content-Type:application/json" --data-binary @src/test/resources/A-Head-Full-of-Dreams.json /document/v1/namespace/music/docid/1 +$ vespa curl -- -v --data-urlencode "yql=select * from music where album contains 'head';" /search/\?hits=5`, + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + target, err := cli.target(targetOptions{}) + if err != nil { + return err + } + service, err := target.Service(curlService, 0, 0, cli.config.cluster()) + if err != nil { + return err + } + url := joinURL(service.BaseURL, args[len(args)-1]) + rawArgs := args[:len(args)-1] + c, err := curl.RawArgs(url, rawArgs...) + if err != nil { + return err + } + switch curlService { + case vespa.DeployService: + if err := addAccessToken(c, target); err != nil { + return err + } + case vespa.DocumentService, vespa.QueryService: + c.PrivateKey = service.TLSOptions.PrivateKeyFile + c.Certificate = service.TLSOptions.CertificateFile + default: + return fmt.Errorf("service not found: %s", curlService) + } + + if dryRun { + log.Print(c.String()) + } else { + if err := c.Run(os.Stdout, os.Stderr); err != nil { + return fmt.Errorf("failed to execute curl: %w", err) + } + } + return nil + }, + } + cmd.Flags().BoolVarP(&dryRun, "dry-run", "n", false, "Print the curl command that would be executed") + cmd.Flags().StringVarP(&curlService, "service", "s", "query", "Which service to query. Must be \"deploy\", \"document\" or \"query\"") + return cmd +} + +func addAccessToken(cmd *curl.Command, target vespa.Target) error { + if target.Type() != vespa.TargetCloud { + return nil + } + req := http.Request{} + if err := target.SignRequest(&req, ""); err != nil { + return err + } + headerValue := req.Header.Get("Authorization") + if headerValue == "" { + return fmt.Errorf("no authorization header added when signing request") + } + cmd.Header("Authorization", headerValue) + return nil +} + +func joinURL(baseURL, path string) string { + baseURL = strings.TrimSuffix(baseURL, "/") + path = strings.TrimPrefix(path, "/") + return baseURL + "/" + path +} diff --git a/client/go/internal/cli/cmd/curl_test.go b/client/go/internal/cli/cmd/curl_test.go new file mode 100644 index 00000000000..3eca0726bb4 --- /dev/null +++ b/client/go/internal/cli/cmd/curl_test.go @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package cmd + +import ( + "fmt" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCurl(t *testing.T) { + cli, stdout, _ := newTestCLI(t) + cli.Environment["VESPA_CLI_ENDPOINTS"] = "{\"endpoints\":[{\"cluster\":\"container\",\"url\":\"http://127.0.0.1:8080\"}]}" + assert.Nil(t, cli.Run("config", "set", "application", "t1.a1.i1")) + assert.Nil(t, cli.Run("config", "set", "target", "cloud")) + assert.Nil(t, cli.Run("config", "set", "cluster", "container")) + assert.Nil(t, cli.Run("auth", "api-key")) + assert.Nil(t, cli.Run("auth", "cert", "--no-add")) + + stdout.Reset() + err := cli.Run("curl", "-n", "--", "-v", "--data-urlencode", "arg=with space", "/search") + assert.Nil(t, err) + + expected := fmt.Sprintf("curl --key %s --cert %s -v --data-urlencode 'arg=with space' http://127.0.0.1:8080/search\n", + filepath.Join(cli.config.homeDir, "t1.a1.i1", "data-plane-private-key.pem"), + filepath.Join(cli.config.homeDir, "t1.a1.i1", "data-plane-public-cert.pem")) + assert.Equal(t, expected, stdout.String()) + + assert.Nil(t, cli.Run("config", "set", "target", "local")) + + stdout.Reset() + err = cli.Run("curl", "-a", "t1.a1.i1", "-s", "deploy", "-n", "/application/v4/tenant/foo") + assert.Nil(t, err) + expected = "curl http://127.0.0.1:19071/application/v4/tenant/foo\n" + assert.Equal(t, expected, stdout.String()) +} diff --git a/client/go/internal/cli/cmd/deploy.go b/client/go/internal/cli/cmd/deploy.go new file mode 100644 index 00000000000..76027744268 --- /dev/null +++ b/client/go/internal/cli/cmd/deploy.go @@ -0,0 +1,199 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa deploy command +// Author: bratseth + +package cmd + +import ( + "fmt" + "io" + "log" + "strconv" + + "github.com/fatih/color" + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/version" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func newDeployCmd(cli *CLI) *cobra.Command { + var ( + logLevelArg string + versionArg string + ) + cmd := &cobra.Command{ + Use: "deploy [application-directory]", + Short: "Deploy (prepare and activate) an application package", + Long: `Deploy (prepare and activate) an application package. + +When this returns successfully the application package has been validated +and activated on config servers. The process of applying it on individual nodes +has started but may not have completed. + +If application directory is not specified, it defaults to working directory. + +When deploying to Vespa Cloud the system can be overridden by setting the +environment variable VESPA_CLI_CLOUD_SYSTEM. This is intended for internal use +only. + +In Vespa Cloud you may override the Vespa runtime version for your deployment. +This option should only be used if you have a reason for using a specific +version. By default Vespa Cloud chooses a suitable version for you. +`, + Example: `$ vespa deploy . +$ vespa deploy -t cloud +$ vespa deploy -t cloud -z dev.aws-us-east-1c # -z can be omitted here as this zone is the default +$ vespa deploy -t cloud -z perf.aws-us-east-1c`, + Args: cobra.MaximumNArgs(1), + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + pkg, err := cli.applicationPackageFrom(args, true) + if err != nil { + return err + } + target, err := cli.target(targetOptions{logLevel: logLevelArg}) + if err != nil { + return err + } + opts, err := cli.createDeploymentOptions(pkg, target) + if err != nil { + return err + } + if versionArg != "" { + version, err := version.Parse(versionArg) + if err != nil { + return err + } + opts.Version = version + } + + var result vespa.PrepareResult + err = cli.spinner(cli.Stderr, "Uploading application package ...", func() error { + result, err = vespa.Deploy(opts) + return err + }) + if err != nil { + return err + } + + log.Println() + if opts.Target.IsCloud() { + cli.printSuccess("Triggered deployment of ", color.CyanString(pkg.Path), " with run ID ", color.CyanString(strconv.FormatInt(result.ID, 10))) + } else { + cli.printSuccess("Deployed ", color.CyanString(pkg.Path)) + printPrepareLog(cli.Stderr, result) + } + if opts.Target.IsCloud() { + log.Printf("\nUse %s for deployment status, or follow this deployment at", color.CyanString("vespa status")) + log.Print(color.CyanString(fmt.Sprintf("%s/tenant/%s/application/%s/%s/instance/%s/job/%s-%s/run/%d", + opts.Target.Deployment().System.ConsoleURL, + opts.Target.Deployment().Application.Tenant, opts.Target.Deployment().Application.Application, opts.Target.Deployment().Zone.Environment, + opts.Target.Deployment().Application.Instance, opts.Target.Deployment().Zone.Environment, opts.Target.Deployment().Zone.Region, + result.ID))) + } + return waitForQueryService(cli, target, result.ID) + }, + } + cmd.Flags().StringVarP(&logLevelArg, "log-level", "l", "error", `Log level for Vespa logs. Must be "error", "warning", "info" or "debug"`) + cmd.Flags().StringVarP(&versionArg, "version", "V", "", `Override the Vespa runtime version to use in Vespa Cloud`) + return cmd +} + +func newPrepareCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "prepare application-directory", + Short: "Prepare an application package for activation", + Args: cobra.MaximumNArgs(1), + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + pkg, err := cli.applicationPackageFrom(args, true) + if err != nil { + return fmt.Errorf("could not find application package: %w", err) + } + target, err := cli.target(targetOptions{}) + if err != nil { + return err + } + opts, err := cli.createDeploymentOptions(pkg, target) + if err != nil { + return err + } + var result vespa.PrepareResult + err = cli.spinner(cli.Stderr, "Uploading application package ...", func() error { + result, err = vespa.Prepare(opts) + return err + }) + if err != nil { + return err + } + if err := cli.config.writeSessionID(vespa.DefaultApplication, result.ID); err != nil { + return fmt.Errorf("could not write session id: %w", err) + } + cli.printSuccess("Prepared ", color.CyanString(pkg.Path), " with session ", result.ID) + printPrepareLog(cli.Stderr, result) + return nil + }, + } +} + +func newActivateCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "activate", + Short: "Activate (deploy) a previously prepared application package", + Args: cobra.MaximumNArgs(1), + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + pkg, err := cli.applicationPackageFrom(args, true) + if err != nil { + return fmt.Errorf("could not find application package: %w", err) + } + sessionID, err := cli.config.readSessionID(vespa.DefaultApplication) + if err != nil { + return fmt.Errorf("could not read session id: %w", err) + } + target, err := cli.target(targetOptions{}) + if err != nil { + return err + } + opts, err := cli.createDeploymentOptions(pkg, target) + if err != nil { + return err + } + err = vespa.Activate(sessionID, opts) + if err != nil { + return err + } + cli.printSuccess("Activated ", color.CyanString(pkg.Path), " with session ", sessionID) + return waitForQueryService(cli, target, sessionID) + }, + } +} + +func waitForQueryService(cli *CLI, target vespa.Target, sessionOrRunID int64) error { + timeout, err := cli.config.timeout() + if err != nil { + return err + } + if timeout > 0 { + log.Println() + _, err := cli.service(target, vespa.QueryService, sessionOrRunID, cli.config.cluster()) + return err + } + return nil +} + +func printPrepareLog(stderr io.Writer, result vespa.PrepareResult) { + for _, entry := range result.LogLines { + level := entry.Level + switch level { + case "ERROR": + level = color.RedString(level) + case "WARNING": + level = color.YellowString(level) + } + fmt.Fprintf(stderr, "%s %s\n", level, entry.Message) + } +} diff --git a/client/go/internal/cli/cmd/deploy/activate.go b/client/go/internal/cli/cmd/deploy/activate.go new file mode 100644 index 00000000000..1f475ff0461 --- /dev/null +++ b/client/go/internal/cli/cmd/deploy/activate.go @@ -0,0 +1,44 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa-deploy command +// Author: arnej + +package deploy + +import ( + "fmt" + "strconv" +) + +// main entry point for vespa-deploy activate + +func RunActivate(opts *Options, args []string) error { + var sessId string + if len(args) == 0 { + sessId = getSessionIdFromFile(opts.Tenant) + } else { + sessId = args[0] + } + src := makeConfigsourceUrl(opts) + url := src + pathPrefix(opts) + "/" + sessId + "/active" + url = addUrlPropertyFromFlag(url, opts.Verbose, "verbose") + url = addUrlPropertyFromOption(url, strconv.Itoa(opts.Timeout), "timeout") + fmt.Printf("Activating session %s using %s\n", sessId, urlWithoutQuery(url)) + output, err := curlPutNothing(url) + if err != nil { + return err + } + var result ActivateResult + code, err := decodeResponse(output, &result) + if err != nil { + return err + } + if code == 200 { + fmt.Println(result.Message) + fmt.Println("Checksum: ", result.Application.Checksum) + fmt.Println("Timestamp: ", result.Deploy.Timestamp) + fmt.Println("Generation:", result.Application.Generation) + } else { + err = fmt.Errorf("Request failed. HTTP status code: %d\n%s", code, result.Message) + } + return err +} diff --git a/client/go/internal/cli/cmd/deploy/cmd.go b/client/go/internal/cli/cmd/deploy/cmd.go new file mode 100644 index 00000000000..c4489d11771 --- /dev/null +++ b/client/go/internal/cli/cmd/deploy/cmd.go @@ -0,0 +1,155 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa-deploy command +// Author: arnej + +package deploy + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/cli/build" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func reallySimpleHelp(cmd *cobra.Command, args []string) { + fmt.Println("Usage: vespa-deploy", cmd.Use) +} + +func NewDeployCmd() *cobra.Command { + var ( + curOptions Options + ) + if err := vespa.LoadDefaultEnv(); err != nil { + util.JustExitWith(err) + } + cobra.EnableCommandSorting = false + cmd := &cobra.Command{ + Use: "vespa-deploy [-h] [-v] [-f] [-t] [-c] [-p] [-z] [-V] [<command>] [args]", + Short: "deploy applications to vespa config server", + Long: `Usage: vespa-deploy [-h] [-v] [-f] [-t] [-c] [-p] [-z] [-V] [<command>] [args] +Supported commands: 'upload', 'prepare', 'activate', 'fetch' and 'help' +Supported options: '-h' (help), '-v' (verbose), '-f' (force/ignore validation errors), '-t' (timeout in seconds), '-p' (config server http port) +Try 'vespa-deploy help <command>' to get more help`, + Version: build.Version, + Args: cobra.MaximumNArgs(2), + CompletionOptions: cobra.CompletionOptions{DisableDefaultCmd: true}, + } + cmd.PersistentFlags().BoolVarP(&curOptions.Verbose, "verbose", "v", false, "show details") + cmd.PersistentFlags().BoolVarP(&curOptions.DryRun, "dryrun", "n", false, "dry-run") + cmd.PersistentFlags().BoolVarP(&curOptions.Force, "force", "f", false, "ignore validation errors") + cmd.PersistentFlags().BoolVarP(&curOptions.Hosted, "hosted", "H", false, "for hosted vespa") + + cmd.PersistentFlags().StringVarP(&curOptions.ServerHost, "server", "c", "", "config server hostname") + cmd.PersistentFlags().IntVarP(&curOptions.PortNumber, "port", "p", 19071, "config server http port") + cmd.PersistentFlags().IntVarP(&curOptions.Timeout, "timeout", "t", 900, "timeout in seconds") + + cmd.PersistentFlags().StringVarP(&curOptions.Tenant, "tenant", "e", "default", "which tentant") + cmd.PersistentFlags().StringVarP(&curOptions.Region, "region", "r", "default", "which region") + cmd.PersistentFlags().StringVarP(&curOptions.Environment, "environment", "E", "prod", "which environment") + cmd.PersistentFlags().StringVarP(&curOptions.Application, "application", "a", "default", "which application") + cmd.PersistentFlags().StringVarP(&curOptions.Instance, "instance", "i", "default", "which instance") + + cmd.PersistentFlags().StringVarP(&curOptions.Rotations, "rotations", "R", "", "which rotations") + cmd.PersistentFlags().StringVarP(&curOptions.VespaVersion, "vespaversion", "V", "", "which vespa version") + + cmd.PersistentFlags().MarkHidden("hosted") + cmd.PersistentFlags().MarkHidden("rotations") + cmd.PersistentFlags().MarkHidden("vespaversion") + + cmd.AddCommand(newUploadCmd(&curOptions)) + cmd.AddCommand(newPrepareCmd(&curOptions)) + cmd.AddCommand(newActivateCmd(&curOptions)) + cmd.AddCommand(newFetchCmd(&curOptions)) + + cmd.InitDefaultHelpCmd() + return cmd +} + +func newUploadCmd(opts *Options) *cobra.Command { + cmd := &cobra.Command{ + Use: "upload <application package>", + Run: func(cmd *cobra.Command, args []string) { + opts.Command = CmdUpload + if opts.Verbose { + trace.AdjustVerbosity(1) + } + trace.Trace("upload with", opts, args) + err := RunUpload(opts, args) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err.Error()) + os.Exit(1) + } + }, + Args: cobra.MaximumNArgs(1), + } + cmd.Flags().StringVarP(&opts.From, "from", "F", "", `where from`) + cmd.SetHelpFunc(reallySimpleHelp) + return cmd +} + +func newPrepareCmd(opts *Options) *cobra.Command { + cmd := &cobra.Command{ + Use: "prepare [<session_id> | <application package>]", + Run: func(cmd *cobra.Command, args []string) { + opts.Command = CmdPrepare + if opts.Verbose { + trace.AdjustVerbosity(1) + } + trace.Trace("prepare with", opts, args) + err := RunPrepare(opts, args) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err.Error()) + os.Exit(1) + } + }, + Args: cobra.MaximumNArgs(1), + } + cmd.SetHelpFunc(reallySimpleHelp) + return cmd +} + +func newActivateCmd(opts *Options) *cobra.Command { + cmd := &cobra.Command{ + Use: "activate [<session_id>]", + Run: func(cmd *cobra.Command, args []string) { + opts.Command = CmdActivate + if opts.Verbose { + trace.AdjustVerbosity(1) + } + trace.Trace("activate with", opts, args) + err := RunActivate(opts, args) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err.Error()) + os.Exit(1) + } + }, + Args: cobra.MaximumNArgs(1), + } + cmd.SetHelpFunc(reallySimpleHelp) + return cmd +} + +func newFetchCmd(opts *Options) *cobra.Command { + cmd := &cobra.Command{ + Use: "fetch <output directory>", + Run: func(cmd *cobra.Command, args []string) { + opts.Command = CmdFetch + if opts.Verbose { + trace.AdjustVerbosity(1) + } + trace.Trace("fetch with", opts, args) + err := RunFetch(opts, args) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err.Error()) + os.Exit(1) + } + }, + Args: cobra.MaximumNArgs(1), + } + cmd.SetHelpFunc(reallySimpleHelp) + return cmd +} diff --git a/client/go/internal/cli/cmd/deploy/curl.go b/client/go/internal/cli/cmd/deploy/curl.go new file mode 100644 index 00000000000..b46d4e361a9 --- /dev/null +++ b/client/go/internal/cli/cmd/deploy/curl.go @@ -0,0 +1,122 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa-deploy command +// Author: arnej + +package deploy + +import ( + "bytes" + "fmt" + "io" + "os" + "os/exec" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/curl" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func curlPutNothing(url string) (string, error) { + cmd := newCurlCommand(url, curlPutArgs()) + cmd.Method = "PUT" + var out bytes.Buffer + err := runCurl(cmd, &out) + return out.String(), err +} + +func curlPost(url string, input io.Reader) (string, error) { + cmd := newCurlCommand(url, curlPostArgs()) + cmd.Method = "POST" + cmd.Header("Content-Type", "application/x-gzip") + cmd.WithBodyInput(input) + var out bytes.Buffer + err := runCurl(cmd, &out) + return out.String(), err +} + +func curlPostZip(url string, input io.Reader) (string, error) { + cmd := newCurlCommand(url, curlPostArgs()) + cmd.Method = "POST" + cmd.Header("Content-Type", "application/zip") + cmd.WithBodyInput(input) + var out bytes.Buffer + err := runCurl(cmd, &out) + return out.String(), err +} + +func curlGet(url string, output io.Writer) error { + cmd := newCurlCommand(url, commonCurlArgs()) + err := runCurl(cmd, output) + return err +} + +func urlWithoutQuery(url string) string { + parts := strings.Split(url, "?") + return parts[0] +} + +func newCurlCommand(url string, args []string) *curl.Command { + tls, err := vespa.LoadTlsConfig() + if err != nil { + util.JustExitWith(err) + } + if tls != nil && strings.HasPrefix(url, "http:") { + url = "https:" + url[5:] + } + cmd, err := curl.RawArgs(url, args...) + if err != nil { + util.JustExitWith(err) + } + if tls != nil { + if tls.DisableHostnameValidation { + cmd, err = curl.RawArgs(url, append(args, "--insecure")...) + if err != nil { + util.JustExitWith(err) + } + } + cmd.PrivateKey = tls.Files.PrivateKey + cmd.Certificate = tls.Files.Certificates + cmd.CaCertificate = tls.Files.CaCertificates + } + return cmd +} + +func runCurl(cmd *curl.Command, stdout io.Writer) error { + trace.Trace("running curl:", cmd.String()) + err := cmd.Run(stdout, os.Stderr) + if err != nil { + if ee, ok := err.(*exec.ExitError); ok { + if ee.ProcessState.ExitCode() == 7 { + return fmt.Errorf("HTTP request failed. Could not connect to %s", cmd.GetUrlPrefix()) + } + } + return fmt.Errorf("HTTP request failed with curl %s", err.Error()) + } + return err +} + +func commonCurlArgs() []string { + return []string{ + "-A", "vespa-deploy", + "--silent", + "--show-error", + "--connect-timeout", "30", + "--max-time", "1200", + } +} + +func curlPutArgs() []string { + return append(commonCurlArgs(), + "--write-out", "\n%{http_code}") +} + +func curlGetArgs() []string { + return commonCurlArgs() +} + +func curlPostArgs() []string { + return append(commonCurlArgs(), + "--write-out", "\n%{http_code}") +} diff --git a/client/go/internal/cli/cmd/deploy/fetch.go b/client/go/internal/cli/cmd/deploy/fetch.go new file mode 100644 index 00000000000..47eeb8631c6 --- /dev/null +++ b/client/go/internal/cli/cmd/deploy/fetch.go @@ -0,0 +1,96 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa-deploy command +// Author: arnej + +package deploy + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +// main entry point for vespa-deploy fetch + +func RunFetch(opts *Options, args []string) error { + dirName := "." + if len(args) > 0 { + dirName = args[0] + } + src := makeConfigsourceUrl(opts) + url := src + + "/application/v2" + + "/tenant/" + opts.Tenant + + "/application/" + opts.Application + + "/environment/" + opts.Environment + + "/region/" + opts.Region + + "/instance/" + opts.Instance + + "/content/" + + url = addUrlPropertyFromOption(url, strconv.Itoa(opts.Timeout), "timeout") + fmt.Printf("Writing active application to %s\n(using %s)\n", dirName, urlWithoutQuery(url)) + var out bytes.Buffer + err := curlGet(url, &out) + if err != nil { + return err + } + fetchDirectory(dirName, &out) + return err +} + +func fetchDirectory(name string, input *bytes.Buffer) { + err := os.MkdirAll(name, 0755) + if err != nil { + fmt.Printf("ERROR: %v\n", err) + return + } + codec := json.NewDecoder(input) + var result []string + err = codec.Decode(&result) + if err != nil { + fmt.Printf("ERROR: %v [%v] <<< %s\n", result, err, input.String()) + return + } + for _, entry := range result { + fmt.Println("GET", entry) + fn := name + "/" + getPartAfterSlash(entry) + if strings.HasSuffix(entry, "/") { + var out bytes.Buffer + err := curlGet(entry, &out) + if err != nil { + fmt.Println("FAILED", err) + return + } + fetchDirectory(fn, &out) + } else { + f, err := os.Create(fn) + if err != nil { + fmt.Println("FAILED", err) + return + } + defer f.Close() + err = curlGet(entry, f) + if err != nil { + fmt.Println("FAILED", err) + return + } + } + } +} + +func getPartAfterSlash(path string) string { + parts := strings.Split(path, "/") + idx := len(parts) - 1 + if idx > 1 && parts[idx] == "" { + return parts[idx-1] + } + if idx == 0 { + util.JustExitMsg("cannot find part after slash: " + path) + } + return parts[idx] +} diff --git a/client/go/internal/cli/cmd/deploy/options.go b/client/go/internal/cli/cmd/deploy/options.go new file mode 100644 index 00000000000..21ee8c902ed --- /dev/null +++ b/client/go/internal/cli/cmd/deploy/options.go @@ -0,0 +1,70 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa-deploy command +// Author: arnej + +package deploy + +import ( + "strconv" + "strings" +) + +type CmdType int + +const ( + CmdNone CmdType = iota + CmdUpload + CmdPrepare + CmdActivate + CmdFetch +) + +type Options struct { + Command CmdType + + Verbose bool + DryRun bool + Force bool + Hosted bool + + Application string + Environment string + From string + Instance string + Region string + Rotations string + ServerHost string + Tenant string + VespaVersion string + + Timeout int + PortNumber int +} + +func (opts *Options) String() string { + var buf strings.Builder + buf.WriteString("command-line options [") + if opts.DryRun { + buf.WriteString(" dry-run") + } + if opts.Force { + buf.WriteString(" force") + } + if opts.Hosted { + buf.WriteString(" hosted") + } + if opts.ServerHost != "" { + buf.WriteString(" server=") + buf.WriteString(opts.ServerHost) + } + if opts.PortNumber != 19071 { + buf.WriteString(" port=") + buf.WriteString(strconv.Itoa(opts.PortNumber)) + } + if opts.From != "" { + buf.WriteString(" from=") + buf.WriteString(opts.From) + } + buf.WriteString(" ]") + return buf.String() +} diff --git a/client/go/internal/cli/cmd/deploy/persist.go b/client/go/internal/cli/cmd/deploy/persist.go new file mode 100644 index 00000000000..e52642693fb --- /dev/null +++ b/client/go/internal/cli/cmd/deploy/persist.go @@ -0,0 +1,85 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa-deploy command +// Author: arnej + +package deploy + +import ( + "os" + "path/filepath" + + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +const ( + cloudconfigDir = ".cloudconfig" + configsourceUrlUsedFileName = "deploy-configsource-url-used" + sessionIdFileName = "deploy-session-id" +) + +func createCloudconfigDir() (string, error) { + userHome, err := os.UserHomeDir() + if err != nil { + return "", err + } + home := filepath.Join(userHome, cloudconfigDir) + if err := os.MkdirAll(home, 0700); err != nil { + return "", err + } + return home, nil +} + +func configsourceUrlUsedFile() string { + home, err := createCloudconfigDir() + if err != nil { + home = "/tmp" + } + return filepath.Join(home, configsourceUrlUsedFileName) +} + +func createTenantDir(tenant string) string { + home, err := createCloudconfigDir() + if err != nil { + util.JustExitWith(err) + } + tdir := filepath.Join(home, tenant) + if err := os.MkdirAll(tdir, 0700); err != nil { + util.JustExitWith(err) + } + return tdir +} + +func writeConfigsourceUrlUsed(url string) { + fn := configsourceUrlUsedFile() + os.WriteFile(fn, []byte(url), 0600) +} + +func getConfigsourceUrlUsed() string { + fn := configsourceUrlUsedFile() + bytes, err := os.ReadFile(fn) + if err != nil { + return "" + } + return string(bytes) +} + +func writeSessionIdToFile(tenant, newSessionId string) { + if newSessionId != "" { + dir := createTenantDir(tenant) + fn := filepath.Join(dir, sessionIdFileName) + os.WriteFile(fn, []byte(newSessionId), 0600) + trace.Trace("wrote", newSessionId, "to", fn) + } +} + +func getSessionIdFromFile(tenant string) string { + dir := createTenantDir(tenant) + fn := filepath.Join(dir, sessionIdFileName) + bytes, err := os.ReadFile(fn) + if err != nil { + util.JustExitMsg("Could not read session id from file, and no session id supplied as argument. Exiting.") + } + trace.Trace("Session-id", string(bytes), "found from file", fn) + return string(bytes) +} diff --git a/client/go/internal/cli/cmd/deploy/prepare.go b/client/go/internal/cli/cmd/deploy/prepare.go new file mode 100644 index 00000000000..4e048883746 --- /dev/null +++ b/client/go/internal/cli/cmd/deploy/prepare.go @@ -0,0 +1,83 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa-deploy command +// Author: arnej + +package deploy + +import ( + "fmt" + "os" + "strconv" +) + +// main entry point for vespa-deploy prepare + +func looksLikeNumber(s string) bool { + var i, j int + n, err := fmt.Sscanf(s+" 123", "%d %d", &i, &j) + return n == 2 && err == nil +} + +func RunPrepare(opts *Options, args []string) (err error) { + var response string + if len(args) == 0 { + // prepare last upload + sessId := getSessionIdFromFile(opts.Tenant) + response, err = doPrepare(opts, sessId) + } else if isFileOrDir(args[0]) { + err := RunUpload(opts, args) + if err != nil { + return err + } + return RunPrepare(opts, []string{}) + } else if looksLikeNumber(args[0]) { + response, err = doPrepare(opts, args[0]) + } else { + err = fmt.Errorf("Command failed. No directory or zip file found: '%s'", args[0]) + } + if err != nil { + return err + } + var result PrepareResult + code, err := decodeResponse(response, &result) + if err != nil { + return err + } + for _, entry := range result.Log { + fmt.Println(entry.Level+":", entry.Message) + } + if code != 200 { + return fmt.Errorf("Request failed. HTTP status code: %d\n%s", code, result.Message) + } + fmt.Println(result.Message) + return err +} + +func isFileOrDir(name string) bool { + f, err := os.Open(name) + if err != nil { + return false + } + st, err := f.Stat() + if err != nil { + return false + } + return st.Mode().IsRegular() || st.Mode().IsDir() +} + +func doPrepare(opts *Options, sessionId string) (output string, err error) { + src := makeConfigsourceUrl(opts) + url := src + pathPrefix(opts) + "/" + sessionId + "/prepared" + url = addUrlPropertyFromFlag(url, opts.Force, "ignoreValidationErrors") + url = addUrlPropertyFromFlag(url, opts.DryRun, "dryRun") + url = addUrlPropertyFromFlag(url, opts.Verbose, "verbose") + url = addUrlPropertyFromFlag(url, opts.Hosted, "hostedVespa") + url = addUrlPropertyFromOption(url, opts.Application, "applicationName") + url = addUrlPropertyFromOption(url, opts.Instance, "instance") + url = addUrlPropertyFromOption(url, strconv.Itoa(opts.Timeout), "timeout") + url = addUrlPropertyFromOption(url, opts.Rotations, "rotations") + url = addUrlPropertyFromOption(url, opts.VespaVersion, "vespaVersion") + fmt.Printf("Preparing session %s using %s\n", sessionId, urlWithoutQuery(url)) + output, err = curlPutNothing(url) + return +} diff --git a/client/go/internal/cli/cmd/deploy/results.go b/client/go/internal/cli/cmd/deploy/results.go new file mode 100644 index 00000000000..47a05e45ab7 --- /dev/null +++ b/client/go/internal/cli/cmd/deploy/results.go @@ -0,0 +1,86 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa-deploy command +// Author: arnej + +package deploy + +import ( + "encoding/json" + "strings" +) + +func decodeResponse(response string, v interface{}) (code int, err error) { + codec := json.NewDecoder(strings.NewReader(response)) + err = codec.Decode(v) + if err != nil { + return + } + err = codec.Decode(&code) + return +} + +type UploadResult struct { + Log []struct { + Time int64 `json:"time"` + Level string `json:"level"` + Message string `json:"message"` + ApplicationPackage bool `json:"applicationPackage"` + } `json:"log"` + Tenant string `json:"tenant"` + SessionID string `json:"session-id"` + Prepared string `json:"prepared"` + Content string `json:"content"` + Message string `json:"message"` + ErrorCode string `json:"error-code"` +} + +type PrepareResult struct { + Log []struct { + Time int64 `json:"time"` + Level string `json:"level"` + Message string `json:"message"` + ApplicationPackage bool `json:"applicationPackage"` + } `json:"log"` + Tenant string `json:"tenant"` + SessionID string `json:"session-id"` + Activate string `json:"activate"` + Message string `json:"message"` + ErrorCode string `json:"error-code"` + /* not used at the moment: + ConfigChangeActions struct { + Restart []struct { + ClusterName string `json:"clusterName"` + ClusterType string `json:"clusterType"` + ServiceType string `json:"serviceType"` + Messages []string `json:"messages"` + Services []struct { + ServiceName string `json:"serviceName"` + ServiceType string `json:"serviceType"` + ConfigID string `json:"configId"` + HostName string `json:"hostName"` + } `json:"services"` + } `json:"restart"` + Refeed []interface{} `json:"refeed"` + Reindex []interface{} `json:"reindex"` + } `json:"configChangeActions"` + */ +} + +type ActivateResult struct { + Deploy struct { + From string `json:"from"` + Timestamp int64 `json:"timestamp"` + InternalRedeploy bool `json:"internalRedeploy"` + } `json:"deploy"` + Application struct { + ID string `json:"id"` + Checksum string `json:"checksum"` + Generation int `json:"generation"` + PreviousActiveGeneration int `json:"previousActiveGeneration"` + } `json:"application"` + Tenant string `json:"tenant"` + SessionID string `json:"session-id"` + Message string `json:"message"` + URL string `json:"url"` + ErrorCode string `json:"error-code"` +} diff --git a/client/go/internal/cli/cmd/deploy/upload.go b/client/go/internal/cli/cmd/deploy/upload.go new file mode 100644 index 00000000000..9e963338bf7 --- /dev/null +++ b/client/go/internal/cli/cmd/deploy/upload.go @@ -0,0 +1,126 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa-deploy command +// Author: arnej + +package deploy + +import ( + "fmt" + "os" + "os/exec" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" +) + +// main entry point for vespa-deploy upload + +func RunUpload(opts *Options, args []string) error { + output, err := doUpload(opts, args) + if err != nil { + return err + } + var result UploadResult + code, err := decodeResponse(output, &result) + if err != nil { + return err + } + if code != 200 { + return fmt.Errorf("Request failed. HTTP status code: %d\n%s", code, result.Message) + } + fmt.Println(result.Message) + writeSessionIdToFile(opts.Tenant, result.SessionID) + return nil +} + +func doUpload(opts *Options, args []string) (result string, err error) { + sources := makeConfigsourceUrls(opts) + for idx, src := range sources { + if idx > 0 { + fmt.Println(err) + fmt.Println("Retrying with another config server") + } + result, err = uploadToConfigSource(opts, src, args) + if err == nil { + writeConfigsourceUrlUsed(src) + return + } + } + return +} + +func uploadToConfigSource(opts *Options, src string, args []string) (string, error) { + if opts.From != "" { + return uploadFrom(opts, src) + } + if len(args) == 0 { + return uploadDirectory(opts, src, ".") + } else { + f, err := os.Open(args[0]) + if err != nil { + return "", fmt.Errorf("Command failed. No such directory found: '%s'", args[0]) + } + defer f.Close() + st, err := f.Stat() + if err != nil { + return "", err + } + if st.Mode().IsRegular() { + if !strings.HasSuffix(args[0], ".zip") { + return "", fmt.Errorf("Application must be a zip file, was '%s'", args[0]) + } + return uploadFile(opts, src, f, args[0]) + } + if st.Mode().IsDir() { + return uploadDirectory(opts, src, args[0]) + } + return "", fmt.Errorf("Bad arg '%s' with FileMode %v", args[0], st.Mode()) + } +} + +func uploadFrom(opts *Options, src string) (string, error) { + url := src + pathPrefix(opts) + url = addUrlPropertyFromOption(url, opts.From, "from") + url = addUrlPropertyFromFlag(url, opts.Verbose, "verbose") + trace.Trace("Upload from URL", opts.From, "using", urlWithoutQuery(url)) + output, err := curlPost(url, nil) + return output, err +} + +func uploadFile(opts *Options, src string, f *os.File, fileName string) (string, error) { + url := src + pathPrefix(opts) + url = addUrlPropertyFromFlag(url, opts.Verbose, "verbose") + fmt.Printf("Uploading application '%s' using %s\n", fileName, urlWithoutQuery(url)) + output, err := curlPostZip(url, f) + return output, err +} + +func uploadDirectory(opts *Options, src string, dirName string) (string, error) { + url := src + pathPrefix(opts) + url = addUrlPropertyFromFlag(url, opts.Verbose, "verbose") + fmt.Printf("Uploading application '%s' using %s\n", dirName, urlWithoutQuery(url)) + tarCmd := tarCommand(dirName) + pipe, err := tarCmd.StdoutPipe() + if err != nil { + return "", err + } + err = tarCmd.Start() + if err != nil { + return "", err + } + output, err := curlPost(url, pipe) + tarCmd.Wait() + return output, err +} + +func tarCommand(dirName string) *exec.Cmd { + args := []string{ + "-C", dirName, + "--dereference", + "--exclude=.[a-zA-Z0-9]*", + "--exclude=ext", + "-czf", "-", + ".", + } + return exec.Command("tar", args...) +} diff --git a/client/go/internal/cli/cmd/deploy/urls.go b/client/go/internal/cli/cmd/deploy/urls.go new file mode 100644 index 00000000000..ff43bbe29d5 --- /dev/null +++ b/client/go/internal/cli/cmd/deploy/urls.go @@ -0,0 +1,73 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa-deploy command +// Author: arnej + +package deploy + +import ( + "fmt" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func makeConfigsourceUrl(opts *Options) string { + src := makeConfigsourceUrls(opts)[0] + if opts.Command == CmdPrepare || opts.Command == CmdActivate { + if lastUsed := getConfigsourceUrlUsed(); lastUsed != "" { + return lastUsed + } + fmt.Printf("Could not read config server URL used for previous upload of an application package, trying to use %s\n", src) + } + return src +} + +func makeConfigsourceUrls(opts *Options) []string { + var results = make([]string, 0, 3) + if opts.ServerHost == "" { + home := vespa.FindHome() + backticks := util.BackTicksForwardStderr + configsources, _ := backticks.Run(home+"/bin/vespa-print-default", "configservers_http") + for _, src := range strings.Split(configsources, "\n") { + colonParts := strings.Split(src, ":") + if len(colonParts) > 1 { + // XXX overwrites port number from above - is this sensible? + src = fmt.Sprintf("%s:%s:%d", colonParts[0], colonParts[1], opts.PortNumber) + trace.Trace("can use config server at", src) + results = append(results, src) + } + } + if len(results) == 0 { + trace.Warning("Could not get url to config server, make sure that VESPA_CONFIGSERVERS is set") + results = append(results, fmt.Sprintf("http://localhost:%d", opts.PortNumber)) + } + } else { + results = append(results, fmt.Sprintf("http://%s:%d", opts.ServerHost, opts.PortNumber)) + } + return results +} + +func pathPrefix(opts *Options) string { + return "/application/v2/tenant/" + opts.Tenant + "/session" +} + +func addUrlPropertyFromFlag(url string, flag bool, propName string) string { + if !flag { + return url + } else { + return addUrlPropertyFromOption(url, "true", propName) + } +} + +func addUrlPropertyFromOption(url, flag, propName string) string { + if flag == "" { + return url + } + if strings.Contains(url, "?") { + return url + "&" + propName + "=" + flag + } else { + return url + "?" + propName + "=" + flag + } +} diff --git a/client/go/internal/cli/cmd/deploy_test.go b/client/go/internal/cli/cmd/deploy_test.go new file mode 100644 index 00000000000..9eaf878bc5e --- /dev/null +++ b/client/go/internal/cli/cmd/deploy_test.go @@ -0,0 +1,199 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// deploy command tests +// Author: bratseth + +package cmd + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/mock" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func TestPrepareZip(t *testing.T) { + assertPrepare("testdata/applications/withTarget/target/application.zip", + []string{"prepare", "testdata/applications/withTarget/target/application.zip"}, t) +} + +func TestActivateZip(t *testing.T) { + assertActivate("testdata/applications/withTarget/target/application.zip", + []string{"activate", "testdata/applications/withTarget/target/application.zip"}, t) +} + +func TestDeployZip(t *testing.T) { + assertDeploy("testdata/applications/withTarget/target/application.zip", + []string{"deploy", "testdata/applications/withTarget/target/application.zip"}, t) +} + +func TestDeployZipWithURLTargetArgument(t *testing.T) { + applicationPackage := "testdata/applications/withTarget/target/application.zip" + arguments := []string{"deploy", "testdata/applications/withTarget/target/application.zip", "-t", "http://target:19071"} + + client := &mock.HTTPClient{} + cli, stdout, _ := newTestCLI(t) + cli.httpClient = client + assert.Nil(t, cli.Run(arguments...)) + assert.Equal(t, + "\nSuccess: Deployed "+applicationPackage+"\n", + stdout.String()) + assertDeployRequestMade("http://target:19071", client, t) +} + +func TestDeployZipWitLocalTargetArgument(t *testing.T) { + assertDeploy("testdata/applications/withTarget/target/application.zip", + []string{"deploy", "testdata/applications/withTarget/target/application.zip", "-t", "local"}, t) +} + +func TestDeploySourceDirectory(t *testing.T) { + assertDeploy("testdata/applications/withSource/src/main/application", + []string{"deploy", "testdata/applications/withSource/src/main/application"}, t) +} + +func TestDeployApplicationDirectoryWithSource(t *testing.T) { + assertDeploy("testdata/applications/withSource/src/main/application", + []string{"deploy", "testdata/applications/withSource"}, t) +} + +func TestDeployApplicationDirectoryWithPomAndTarget(t *testing.T) { + assertDeploy("testdata/applications/withTarget/target/application.zip", + []string{"deploy", "testdata/applications/withTarget"}, t) +} + +func TestDeployApplicationDirectoryWithPomAndEmptyTarget(t *testing.T) { + cli, _, stderr := newTestCLI(t) + assert.NotNil(t, cli.Run("deploy", "testdata/applications/withEmptyTarget")) + assert.Equal(t, + "Error: found pom.xml, but target/application.zip does not exist: run 'mvn package' first\n", + stderr.String()) +} + +func TestDeployApplicationPackageErrorWithUnexpectedNonJson(t *testing.T) { + assertApplicationPackageError(t, "deploy", 401, + "Raw text error", + "Raw text error") +} + +func TestDeployApplicationPackageErrorWithUnexpectedJson(t *testing.T) { + assertApplicationPackageError(t, "deploy", 401, + `{ + "some-unexpected-json": "Invalid XML, error in services.xml: element \"nosuch\" not allowed here" +}`, + `{"some-unexpected-json": "Invalid XML, error in services.xml: element \"nosuch\" not allowed here"}`) +} + +func TestDeployApplicationPackageErrorWithExpectedFormat(t *testing.T) { + assertApplicationPackageError(t, "deploy", 400, + "Invalid XML, error in services.xml:\nelement \"nosuch\" not allowed here", + `{ + "error-code": "INVALID_APPLICATION_PACKAGE", + "message": "Invalid XML, error in services.xml: element \"nosuch\" not allowed here" + }`) +} + +func TestPrepareApplicationPackageErrorWithExpectedFormat(t *testing.T) { + assertApplicationPackageError(t, "prepare", 400, + "Invalid XML, error in services.xml:\nelement \"nosuch\" not allowed here", + `{ + "error-code": "INVALID_APPLICATION_PACKAGE", + "message": "Invalid XML, error in services.xml: element \"nosuch\" not allowed here" + }`) +} + +func TestDeployError(t *testing.T) { + assertDeployServerError(t, 501, "Deploy service error") +} + +func assertDeploy(applicationPackage string, arguments []string, t *testing.T) { + t.Helper() + cli, stdout, _ := newTestCLI(t) + client := &mock.HTTPClient{} + cli.httpClient = client + assert.Nil(t, cli.Run(arguments...)) + assert.Equal(t, + "\nSuccess: Deployed "+applicationPackage+"\n", + stdout.String()) + assertDeployRequestMade("http://127.0.0.1:19071", client, t) +} + +func assertPrepare(applicationPackage string, arguments []string, t *testing.T) { + t.Helper() + client := &mock.HTTPClient{} + client.NextResponseString(200, `{"session-id":"42"}`) + client.NextResponseString(200, `{"session-id":"42","message":"Session 42 for tenant 'default' prepared.","log":[{"level":"WARNING","message":"Warning message 1","time": 1430134091319}]}`) + cli, stdout, _ := newTestCLI(t) + cli.httpClient = client + assert.Nil(t, cli.Run(arguments...)) + assert.Equal(t, + "Success: Prepared "+applicationPackage+" with session 42\n", + stdout.String()) + + assertPackageUpload(0, "http://127.0.0.1:19071/application/v2/tenant/default/session", client, t) + sessionURL := "http://127.0.0.1:19071/application/v2/tenant/default/session/42/prepared" + assert.Equal(t, sessionURL, client.Requests[1].URL.String()) + assert.Equal(t, "PUT", client.Requests[1].Method) +} + +func assertActivate(applicationPackage string, arguments []string, t *testing.T) { + t.Helper() + client := &mock.HTTPClient{} + cli, stdout, _ := newTestCLI(t) + cli.httpClient = client + if err := cli.config.writeSessionID(vespa.DefaultApplication, 42); err != nil { + t.Fatal(err) + } + assert.Nil(t, cli.Run(arguments...)) + assert.Equal(t, + "Success: Activated "+applicationPackage+" with session 42\n", + stdout.String()) + url := "http://127.0.0.1:19071/application/v2/tenant/default/session/42/active" + assert.Equal(t, url, client.LastRequest.URL.String()) + assert.Equal(t, "PUT", client.LastRequest.Method) +} + +func assertPackageUpload(requestNumber int, url string, client *mock.HTTPClient, t *testing.T) { + t.Helper() + req := client.LastRequest + if requestNumber >= 0 { + req = client.Requests[requestNumber] + } + assert.Equal(t, url, req.URL.String()) + assert.Equal(t, "application/zip", req.Header.Get("Content-Type")) + assert.Equal(t, "POST", req.Method) + var body = req.Body + assert.NotNil(t, body) + buf := make([]byte, 7) // Just check the first few bytes + body.Read(buf) + assert.Equal(t, "PK\x03\x04\x14\x00\b", string(buf)) +} + +func assertDeployRequestMade(target string, client *mock.HTTPClient, t *testing.T) { + t.Helper() + assertPackageUpload(-1, target+"/application/v2/tenant/default/prepareandactivate", client, t) +} + +func assertApplicationPackageError(t *testing.T, cmd string, status int, expectedMessage string, returnBody string) { + t.Helper() + client := &mock.HTTPClient{} + client.NextResponseString(status, returnBody) + cli, _, stderr := newTestCLI(t) + cli.httpClient = client + assert.NotNil(t, cli.Run(cmd, "testdata/applications/withTarget/target/application.zip")) + assert.Equal(t, + "Error: invalid application package (Status "+strconv.Itoa(status)+")\n"+expectedMessage+"\n", + stderr.String()) +} + +func assertDeployServerError(t *testing.T, status int, errorMessage string) { + t.Helper() + client := &mock.HTTPClient{} + client.NextResponseString(status, errorMessage) + cli, _, stderr := newTestCLI(t) + cli.httpClient = client + assert.NotNil(t, cli.Run("deploy", "testdata/applications/withTarget/target/application.zip")) + assert.Equal(t, + "Error: error from deploy api at 127.0.0.1:19071 (Status "+strconv.Itoa(status)+"):\n"+errorMessage+"\n", + stderr.String()) +} diff --git a/client/go/internal/cli/cmd/document.go b/client/go/internal/cli/cmd/document.go new file mode 100644 index 00000000000..b5b63fd32df --- /dev/null +++ b/client/go/internal/cli/cmd/document.go @@ -0,0 +1,224 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa document command +// author: bratseth + +package cmd + +import ( + "fmt" + "io" + "strings" + "time" + + "github.com/fatih/color" + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func addDocumentFlags(cmd *cobra.Command, printCurl *bool, timeoutSecs *int) { + cmd.PersistentFlags().BoolVarP(printCurl, "verbose", "v", false, "Print the equivalent curl command for the document operation") + cmd.PersistentFlags().IntVarP(timeoutSecs, "timeout", "T", 60, "Timeout for the document request in seconds") +} + +func newDocumentCmd(cli *CLI) *cobra.Command { + var ( + printCurl bool + timeoutSecs int + ) + cmd := &cobra.Command{ + Use: "document json-file", + Short: "Issue a document operation to Vespa", + Long: `Issue a document operation to Vespa. + +The operation must be on the format documented in +https://docs.vespa.ai/en/reference/document-json-format.html#document-operations + +When this returns successfully, the document is guaranteed to be visible in any +subsequent get or query operation. + +To feed with high throughput, https://docs.vespa.ai/en/vespa-feed-client.html +should be used instead of this.`, + Example: `$ vespa document src/test/resources/A-Head-Full-of-Dreams.json`, + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + service, err := documentService(cli) + if err != nil { + return err + } + return printResult(cli, vespa.Send(args[0], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), false) + }, + } + addDocumentFlags(cmd, &printCurl, &timeoutSecs) + return cmd +} + +func newDocumentPutCmd(cli *CLI) *cobra.Command { + var ( + printCurl bool + timeoutSecs int + ) + cmd := &cobra.Command{ + Use: "put [id] json-file", + Short: "Writes a document to Vespa", + Long: `Writes the document in the given file to Vespa. +If the document already exists, all its values will be replaced by this document. +If the document id is specified both as an argument and in the file the argument takes precedence.`, + Args: cobra.RangeArgs(1, 2), + Example: `$ vespa document put src/test/resources/A-Head-Full-of-Dreams.json +$ vespa document put id:mynamespace:music::a-head-full-of-dreams src/test/resources/A-Head-Full-of-Dreams.json`, + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + service, err := documentService(cli) + if err != nil { + return err + } + if len(args) == 1 { + return printResult(cli, vespa.Put("", args[0], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), false) + } else { + return printResult(cli, vespa.Put(args[0], args[1], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), false) + } + }, + } + addDocumentFlags(cmd, &printCurl, &timeoutSecs) + return cmd +} + +func newDocumentUpdateCmd(cli *CLI) *cobra.Command { + var ( + printCurl bool + timeoutSecs int + ) + cmd := &cobra.Command{ + Use: "update [id] json-file", + Short: "Modifies some fields of an existing document", + Long: `Updates the values of the fields given in a json file as specified in the file. +If the document id is specified both as an argument and in the file the argument takes precedence.`, + Args: cobra.RangeArgs(1, 2), + Example: `$ vespa document update src/test/resources/A-Head-Full-of-Dreams-Update.json +$ vespa document update id:mynamespace:music::a-head-full-of-dreams src/test/resources/A-Head-Full-of-Dreams.json`, + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + service, err := documentService(cli) + if err != nil { + return err + } + if len(args) == 1 { + return printResult(cli, vespa.Update("", args[0], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), false) + } else { + return printResult(cli, vespa.Update(args[0], args[1], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), false) + } + }, + } + addDocumentFlags(cmd, &printCurl, &timeoutSecs) + return cmd +} + +func newDocumentRemoveCmd(cli *CLI) *cobra.Command { + var ( + printCurl bool + timeoutSecs int + ) + cmd := &cobra.Command{ + Use: "remove id | json-file", + Short: "Removes a document from Vespa", + Long: `Removes the document specified either as a document id or given in the json file. +If the document id is specified both as an argument and in the file the argument takes precedence.`, + Args: cobra.ExactArgs(1), + Example: `$ vespa document remove src/test/resources/A-Head-Full-of-Dreams-Remove.json +$ vespa document remove id:mynamespace:music::a-head-full-of-dreams`, + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + service, err := documentService(cli) + if err != nil { + return err + } + if strings.HasPrefix(args[0], "id:") { + return printResult(cli, vespa.RemoveId(args[0], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), false) + } else { + return printResult(cli, vespa.RemoveOperation(args[0], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), false) + } + }, + } + addDocumentFlags(cmd, &printCurl, &timeoutSecs) + return cmd +} + +func newDocumentGetCmd(cli *CLI) *cobra.Command { + var ( + printCurl bool + timeoutSecs int + ) + cmd := &cobra.Command{ + Use: "get id", + Short: "Gets a document", + Args: cobra.ExactArgs(1), + DisableAutoGenTag: true, + SilenceUsage: true, + Example: `$ vespa document get id:mynamespace:music::a-head-full-of-dreams`, + RunE: func(cmd *cobra.Command, args []string) error { + service, err := documentService(cli) + if err != nil { + return err + } + return printResult(cli, vespa.Get(args[0], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), true) + }, + } + addDocumentFlags(cmd, &printCurl, &timeoutSecs) + return cmd +} + +func documentService(cli *CLI) (*vespa.Service, error) { + target, err := cli.target(targetOptions{}) + if err != nil { + return nil, err + } + return cli.service(target, vespa.DocumentService, 0, cli.config.cluster()) +} + +func operationOptions(stderr io.Writer, printCurl bool, timeoutSecs int) vespa.OperationOptions { + curlOutput := io.Discard + if printCurl { + curlOutput = stderr + } + return vespa.OperationOptions{ + CurlOutput: curlOutput, + Timeout: time.Second * time.Duration(timeoutSecs), + } +} + +func printResult(cli *CLI, result util.OperationResult, payloadOnlyOnSuccess bool) error { + out := cli.Stdout + if !result.Success { + out = cli.Stderr + } + + if !result.Success { + fmt.Fprintln(out, color.RedString("Error:"), result.Message) + } else if !payloadOnlyOnSuccess || result.Payload == "" { + fmt.Fprintln(out, color.GreenString("Success:"), result.Message) + } + + if result.Detail != "" { + fmt.Fprintln(out, color.YellowString(result.Detail)) + } + + if result.Payload != "" { + if !payloadOnlyOnSuccess { + fmt.Fprintln(out) + } + fmt.Fprintln(out, result.Payload) + } + + if !result.Success { + err := errHint(fmt.Errorf("document operation failed")) + err.quiet = true + return err + } + return nil +} diff --git a/client/go/internal/cli/cmd/document_test.go b/client/go/internal/cli/cmd/document_test.go new file mode 100644 index 00000000000..bf9cc0404dc --- /dev/null +++ b/client/go/internal/cli/cmd/document_test.go @@ -0,0 +1,179 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// document command tests +// Author: bratseth + +package cmd + +import ( + "os" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/mock" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func TestDocumentSendPut(t *testing.T) { + assertDocumentSend([]string{"document", "testdata/A-Head-Full-of-Dreams-Put.json"}, + "put", "POST", "id:mynamespace:music::a-head-full-of-dreams", "testdata/A-Head-Full-of-Dreams-Put.json", t) +} + +func TestDocumentSendPutVerbose(t *testing.T) { + assertDocumentSend([]string{"document", "-v", "testdata/A-Head-Full-of-Dreams-Put.json"}, + "put", "POST", "id:mynamespace:music::a-head-full-of-dreams", "testdata/A-Head-Full-of-Dreams-Put.json", t) +} + +func TestDocumentSendUpdate(t *testing.T) { + assertDocumentSend([]string{"document", "testdata/A-Head-Full-of-Dreams-Update.json"}, + "update", "PUT", "id:mynamespace:music::a-head-full-of-dreams", "testdata/A-Head-Full-of-Dreams-Update.json", t) +} + +func TestDocumentSendRemove(t *testing.T) { + assertDocumentSend([]string{"document", "testdata/A-Head-Full-of-Dreams-Remove.json"}, + "remove", "DELETE", "id:mynamespace:music::a-head-full-of-dreams", "testdata/A-Head-Full-of-Dreams-Remove.json", t) +} + +func TestDocumentPutWithIdArg(t *testing.T) { + assertDocumentSend([]string{"document", "put", "id:mynamespace:music::a-head-full-of-dreams", "testdata/A-Head-Full-of-Dreams-Without-Operation.json"}, + "put", "POST", "id:mynamespace:music::a-head-full-of-dreams", "testdata/A-Head-Full-of-Dreams-Without-Operation.json", t) +} + +func TestDocumentPutWithoutIdArg(t *testing.T) { + assertDocumentSend([]string{"document", "put", "testdata/A-Head-Full-of-Dreams-Put.json"}, + "put", "POST", "id:mynamespace:music::a-head-full-of-dreams", "testdata/A-Head-Full-of-Dreams-Put.json", t) +} + +func TestDocumentUpdateWithIdArg(t *testing.T) { + assertDocumentSend([]string{"document", "update", "id:mynamespace:music::a-head-full-of-dreams", "testdata/A-Head-Full-of-Dreams-Without-Operation.json"}, + "update", "PUT", "id:mynamespace:music::a-head-full-of-dreams", "testdata/A-Head-Full-of-Dreams-Without-Operation.json", t) +} + +func TestDocumentUpdateWithoutIdArg(t *testing.T) { + assertDocumentSend([]string{"document", "update", "testdata/A-Head-Full-of-Dreams-Update.json"}, + "update", "PUT", "id:mynamespace:music::a-head-full-of-dreams", "testdata/A-Head-Full-of-Dreams-Update.json", t) +} + +func TestDocumentRemoveWithIdArg(t *testing.T) { + assertDocumentSend([]string{"document", "remove", "id:mynamespace:music::a-head-full-of-dreams"}, + "remove", "DELETE", "id:mynamespace:music::a-head-full-of-dreams", "testdata/A-Head-Full-of-Dreams-Remove.json", t) +} + +func TestDocumentRemoveWithoutIdArg(t *testing.T) { + assertDocumentSend([]string{"document", "remove", "testdata/A-Head-Full-of-Dreams-Remove.json"}, + "remove", "DELETE", "id:mynamespace:music::a-head-full-of-dreams", "testdata/A-Head-Full-of-Dreams-Remove.json", t) +} + +func TestDocumentSendMissingId(t *testing.T) { + cli, _, stderr := newTestCLI(t) + assert.NotNil(t, cli.Run("document", "put", "testdata/A-Head-Full-of-Dreams-Without-Operation.json")) + assert.Equal(t, + "Error: No document id given neither as argument or as a 'put' key in the json file\n", + stderr.String()) +} + +func TestDocumentSendWithDisagreeingOperations(t *testing.T) { + cli, _, stderr := newTestCLI(t) + assert.NotNil(t, cli.Run("document", "update", "testdata/A-Head-Full-of-Dreams-Put.json")) + assert.Equal(t, + "Error: Wanted document operation is update but the JSON file specifies put\n", + stderr.String()) +} + +func TestDocumentPutDocumentError(t *testing.T) { + assertDocumentError(t, 401, "Document error") +} + +func TestDocumentPutServerError(t *testing.T) { + assertDocumentServerError(t, 501, "Server error") +} + +func TestDocumentGet(t *testing.T) { + assertDocumentGet([]string{"document", "get", "id:mynamespace:music::a-head-full-of-dreams"}, + "id:mynamespace:music::a-head-full-of-dreams", t) +} + +func assertDocumentSend(arguments []string, expectedOperation string, expectedMethod string, expectedDocumentId string, expectedPayloadFile string, t *testing.T) { + client := &mock.HTTPClient{} + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = client + documentURL, err := documentServiceURL(client) + if err != nil { + t.Fatal(err) + } + expectedPath, _ := vespa.IdToURLPath(expectedDocumentId) + expectedURL := documentURL + "/document/v1/" + expectedPath + + assert.Nil(t, cli.Run(arguments...)) + verbose := false + for _, a := range arguments { + if a == "-v" { + verbose = true + } + } + if verbose { + expectedCurl := "curl -X " + expectedMethod + " -H 'Content-Type: application/json' --data-binary @" + expectedPayloadFile + " " + expectedURL + "\n" + assert.Equal(t, expectedCurl, stderr.String()) + } + assert.Equal(t, "Success: "+expectedOperation+" "+expectedDocumentId+"\n", stdout.String()) + assert.Equal(t, expectedURL, client.LastRequest.URL.String()) + assert.Equal(t, "application/json", client.LastRequest.Header.Get("Content-Type")) + assert.Equal(t, expectedMethod, client.LastRequest.Method) + + expectedPayload, _ := os.ReadFile(expectedPayloadFile) + assert.Equal(t, string(expectedPayload), util.ReaderToString(client.LastRequest.Body)) +} + +func assertDocumentGet(arguments []string, documentId string, t *testing.T) { + client := &mock.HTTPClient{} + documentURL, err := documentServiceURL(client) + if err != nil { + t.Fatal(err) + } + client.NextResponseString(200, "{\"fields\":{\"foo\":\"bar\"}}") + cli, stdout, _ := newTestCLI(t) + cli.httpClient = client + assert.Nil(t, cli.Run(arguments...)) + assert.Equal(t, + `{ + "fields": { + "foo": "bar" + } +} +`, + stdout.String()) + expectedPath, _ := vespa.IdToURLPath(documentId) + assert.Equal(t, documentURL+"/document/v1/"+expectedPath, client.LastRequest.URL.String()) + assert.Equal(t, "GET", client.LastRequest.Method) +} + +func assertDocumentError(t *testing.T, status int, errorMessage string) { + client := &mock.HTTPClient{} + client.NextResponseString(status, errorMessage) + cli, _, stderr := newTestCLI(t) + cli.httpClient = client + assert.NotNil(t, cli.Run("document", "put", + "id:mynamespace:music::a-head-full-of-dreams", + "testdata/A-Head-Full-of-Dreams-Put.json")) + assert.Equal(t, + "Error: Invalid document operation: Status "+strconv.Itoa(status)+"\n\n"+errorMessage+"\n", + stderr.String()) +} + +func assertDocumentServerError(t *testing.T, status int, errorMessage string) { + client := &mock.HTTPClient{} + client.NextResponseString(status, errorMessage) + cli, _, stderr := newTestCLI(t) + cli.httpClient = client + assert.NotNil(t, cli.Run("document", "put", + "id:mynamespace:music::a-head-full-of-dreams", + "testdata/A-Head-Full-of-Dreams-Put.json")) + assert.Equal(t, + "Error: Container (document API) at 127.0.0.1:8080: Status "+strconv.Itoa(status)+"\n\n"+errorMessage+"\n", + stderr.String()) +} + +func documentServiceURL(client *mock.HTTPClient) (string, error) { + return "http://127.0.0.1:8080", nil +} diff --git a/client/go/internal/cli/cmd/log.go b/client/go/internal/cli/cmd/log.go new file mode 100644 index 00000000000..fa07e33538c --- /dev/null +++ b/client/go/internal/cli/cmd/log.go @@ -0,0 +1,106 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package cmd + +import ( + "fmt" + "time" + + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func newLogCmd(cli *CLI) *cobra.Command { + var ( + fromArg string + toArg string + levelArg string + followArg bool + dequoteArg bool + ) + cmd := &cobra.Command{ + Use: "log [relative-period]", + Short: "Show the Vespa log", + Long: `Show the Vespa log. + +The logs shown can be limited to a relative or fixed period. All timestamps are shown in UTC. + +Logs for the past hour are shown if no arguments are given. +`, + Example: `$ vespa log 1h +$ vespa log --nldequote=false 10m +$ vespa log --from 2021-08-25T15:00:00Z --to 2021-08-26T02:00:00Z +$ vespa log --follow`, + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + target, err := cli.target(targetOptions{logLevel: levelArg}) + if err != nil { + return err + } + options := vespa.LogOptions{ + Level: vespa.LogLevel(levelArg), + Follow: followArg, + Writer: cli.Stdout, + Dequote: dequoteArg, + } + if options.Follow { + if fromArg != "" || toArg != "" || len(args) > 0 { + return fmt.Errorf("cannot combine --from/--to or relative time with --follow") + } + options.From = time.Now().Add(-5 * time.Minute) + } else { + from, to, err := parsePeriod(fromArg, toArg, args) + if err != nil { + return fmt.Errorf("invalid period: %w", err) + } + options.From = from + options.To = to + } + if err := target.PrintLog(options); err != nil { + return fmt.Errorf("could not retrieve logs: %w", err) + } + return nil + }, + } + cmd.Flags().StringVarP(&fromArg, "from", "F", "", "Include logs since this timestamp (RFC3339 format)") + cmd.Flags().StringVarP(&toArg, "to", "T", "", "Include logs until this timestamp (RFC3339 format)") + cmd.Flags().StringVarP(&levelArg, "level", "l", "debug", `The maximum log level to show. Must be "error", "warning", "info" or "debug"`) + cmd.Flags().BoolVarP(&followArg, "follow", "f", false, "Follow logs") + cmd.Flags().BoolVarP(&dequoteArg, "nldequote", "n", true, "Dequote LF and TAB characters in log messages") + return cmd +} + +func parsePeriod(from, to string, args []string) (time.Time, time.Time, error) { + relativePeriod := from == "" || to == "" + if relativePeriod { + period := "1h" + if len(args) > 0 { + period = args[0] + } + d, err := time.ParseDuration(period) + if err != nil { + return time.Time{}, time.Time{}, err + } + if d > 0 { + d = -d + } + to := time.Now() + from := to.Add(d) + return from, to, nil + } else if len(args) > 0 { + return time.Time{}, time.Time{}, fmt.Errorf("cannot combine --from/--to with relative value: %s", args[0]) + } + t1, err := time.Parse(time.RFC3339, from) + if err != nil { + return time.Time{}, time.Time{}, err + } + t2, err := time.Parse(time.RFC3339, to) + if err != nil { + return time.Time{}, time.Time{}, err + } + if !t2.After(t1) { + return time.Time{}, time.Time{}, fmt.Errorf("--to must specify a time after --from") + } + return t1, t2, nil +} diff --git a/client/go/internal/cli/cmd/log_test.go b/client/go/internal/cli/cmd/log_test.go new file mode 100644 index 00000000000..8727c82fede --- /dev/null +++ b/client/go/internal/cli/cmd/log_test.go @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/mock" + "github.com/vespa-engine/vespa/client/go/internal/version" +) + +func TestLog(t *testing.T) { + _, pkgDir := mock.ApplicationPackageDir(t, false, false) + httpClient := &mock.HTTPClient{} + httpClient.NextResponseString(200, `1632738690.905535 host1a.dev.aws-us-east-1c 806/53 logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication info Switching to the latest deployed set of configurations and components. Application config generation: 52532`) + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = httpClient + + assert.Nil(t, cli.Run("config", "set", "application", "t1.a1.i1")) + assert.Nil(t, cli.Run("config", "set", "target", "cloud")) + assert.Nil(t, cli.Run("auth", "api-key")) + assert.Nil(t, cli.Run("auth", "cert", pkgDir)) + + stdout.Reset() + assert.Nil(t, cli.Run("log", "--from", "2021-09-27T10:00:00Z", "--to", "2021-09-27T11:00:00Z")) + expected := "[2021-09-27 10:31:30.905535] host1a.dev.aws-us-east-1c info logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication Switching to the latest deployed set of configurations and components. Application config generation: 52532\n" + assert.Equal(t, expected, stdout.String()) + + assert.NotNil(t, cli.Run("log", "--from", "2021-09-27T13:12:49Z", "--to", "2021-09-27T13:15:00", "1h")) + assert.Contains(t, stderr.String(), "Error: invalid period: cannot combine --from/--to with relative value: 1h\n") +} + +func TestLogOldClient(t *testing.T) { + cli, _, stderr := newTestCLI(t) + cli.version = version.MustParse("7.0.0") + + _, pkgDir := mock.ApplicationPackageDir(t, false, false) + httpClient := &mock.HTTPClient{} + httpClient.NextResponseString(200, `{"minVersion": "8.0.0"}`) + httpClient.NextResponseString(200, `1632738690.905535 host1a.dev.aws-us-east-1c 806/53 logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication info Switching to the latest deployed set of configurations and components. Application config generation: 52532`) + cli.httpClient = httpClient + + assert.Nil(t, cli.Run("config", "set", "application", "t1.a1.i1")) + assert.Nil(t, cli.Run("config", "set", "target", "cloud")) + assert.Nil(t, cli.Run("auth", "api-key")) + assert.Nil(t, cli.Run("auth", "cert", pkgDir)) + + assert.Nil(t, cli.Run("log")) + expected := "Warning: client version 7.0.0 is less than the minimum supported version: 8.0.0\nHint: This version may not work as expected\nHint: Try 'vespa version' to check for a new version\n" + assert.Contains(t, stderr.String(), expected) +} diff --git a/client/go/internal/cli/cmd/login.go b/client/go/internal/cli/cmd/login.go new file mode 100644 index 00000000000..aa6a18b3b38 --- /dev/null +++ b/client/go/internal/cli/cmd/login.go @@ -0,0 +1,113 @@ +package cmd + +import ( + "fmt" + "log" + "os" + "strings" + "time" + + "github.com/pkg/browser" + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/cli/auth" + "github.com/vespa-engine/vespa/client/go/internal/cli/auth/auth0" +) + +// newLoginCmd runs the login flow guiding the user through the process +// by showing the login instructions, opening the browser. +// Use `expired` to run the login from other commands setup: +// this will only affect the messages. +func newLoginCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "login", + Args: cobra.NoArgs, + Short: "Authenticate the Vespa CLI", + Example: "$ vespa auth login", + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + targetType, err := cli.config.targetType() + if err != nil { + return err + } + system, err := cli.system(targetType) + if err != nil { + return err + } + a, err := auth0.New(cli.config.authConfigPath(), system.Name, system.URL) + if err != nil { + return err + } + state, err := a.Authenticator.Start(ctx) + if err != nil { + return fmt.Errorf("could not start the authentication process: %w", err) + } + + log.Printf("Your Device Confirmation code is: %s\n", state.UserCode) + + auto_open := confirm(cli, "Automatically open confirmation page in your default browser?") + + if auto_open { + log.Printf("Opened link in your browser: %s\n", state.VerificationURI) + err = browser.OpenURL(state.VerificationURI) + if err != nil { + log.Println("Couldn't open the URL, please do it manually") + } + } else { + log.Printf("Please open link in your browser: %s\n", state.VerificationURI) + } + + var res auth.Result + err = cli.spinner(os.Stderr, "Waiting for login to complete in browser ...", func() error { + res, err = a.Authenticator.Wait(ctx, state) + return err + }) + + if err != nil { + return fmt.Errorf("login error: %w", err) + } + + log.Print("\n") + log.Println("Successfully logged in.") + log.Print("\n") + + // store the refresh token + secretsStore := &auth.Keyring{} + err = secretsStore.Set(auth.SecretsNamespace, system.Name, res.RefreshToken) + if err != nil { + // log the error but move on + log.Println("Could not store the refresh token locally, please expect to login again once your access token expired.") + } + + creds := auth0.Credentials{ + AccessToken: res.AccessToken, + ExpiresAt: time.Now().Add(time.Duration(res.ExpiresIn) * time.Second), + Scopes: auth.RequiredScopes(), + } + if err := a.WriteCredentials(creds); err != nil { + return fmt.Errorf("failed to write credentials: %w", err) + } + return err + }, + } +} + +func confirm(cli *CLI, question string) bool { + for { + var answer string + + fmt.Fprintf(cli.Stdout, "%s [Y/n] ", question) + fmt.Fscanln(cli.Stdin, &answer) + + answer = strings.TrimSpace(strings.ToLower(answer)) + + if answer == "y" || answer == "" { + return true + } else if answer == "n" { + return false + } else { + log.Printf("Please answer Y or N.\n") + } + } +} diff --git a/client/go/internal/cli/cmd/logout.go b/client/go/internal/cli/cmd/logout.go new file mode 100644 index 00000000000..f2ee6e87ac7 --- /dev/null +++ b/client/go/internal/cli/cmd/logout.go @@ -0,0 +1,41 @@ +package cmd + +import ( + "log" + + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/cli/auth/auth0" +) + +func newLogoutCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "logout", + Args: cobra.NoArgs, + Short: "Log out of Vespa Cli", + Example: "$ vespa auth logout", + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + targetType, err := cli.config.targetType() + if err != nil { + return err + } + system, err := cli.system(targetType) + if err != nil { + return err + } + a, err := auth0.New(cli.config.authConfigPath(), system.Name, system.URL) + if err != nil { + return err + } + if err := a.RemoveCredentials(); err != nil { + return err + } + + log.Print("\n") + log.Println("Successfully logged out.") + log.Print("\n") + return nil + }, + } +} diff --git a/client/go/internal/cli/cmd/man.go b/client/go/internal/cli/cmd/man.go new file mode 100644 index 00000000000..4d139adb244 --- /dev/null +++ b/client/go/internal/cli/cmd/man.go @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" + "github.com/spf13/cobra/doc" +) + +func newManCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "man directory", + Short: "Generate man pages and write them to given directory", + Args: cobra.ExactArgs(1), + Hidden: true, // Not intended to be called by users + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + dir := args[0] + err := doc.GenManTree(cli.cmd, nil, dir) + if err != nil { + return fmt.Errorf("failed to write man pages: %w", err) + } + cli.printSuccess("Man pages written to ", dir) + return nil + }, + } +} diff --git a/client/go/internal/cli/cmd/man_test.go b/client/go/internal/cli/cmd/man_test.go new file mode 100644 index 00000000000..0c214698047 --- /dev/null +++ b/client/go/internal/cli/cmd/man_test.go @@ -0,0 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package cmd + +import ( + "fmt" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +func TestMan(t *testing.T) { + tmpDir := t.TempDir() + cli, stdout, _ := newTestCLI(t) + assert.Nil(t, cli.Run("man", tmpDir)) + assert.Equal(t, fmt.Sprintf("Success: Man pages written to %s\n", tmpDir), stdout.String()) + assert.True(t, util.PathExists(filepath.Join(tmpDir, "vespa.1"))) +} diff --git a/client/go/internal/cli/cmd/prod.go b/client/go/internal/cli/cmd/prod.go new file mode 100644 index 00000000000..57b7abe5b6e --- /dev/null +++ b/client/go/internal/cli/cmd/prod.go @@ -0,0 +1,410 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package cmd + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" + + "github.com/fatih/color" + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" + "github.com/vespa-engine/vespa/client/go/internal/vespa/xml" +) + +func newProdCmd() *cobra.Command { + return &cobra.Command{ + Use: "prod", + Short: "Deploy an application package to production in Vespa Cloud", + Long: `Deploy an application package to production in Vespa Cloud. + +Configure and deploy your application package to production in Vespa Cloud.`, + Example: `$ vespa prod init +$ vespa prod submit`, + DisableAutoGenTag: true, + SilenceUsage: false, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return fmt.Errorf("invalid command: %s", args[0]) + }, + } +} + +func newProdInitCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "init", + Short: "Modify service.xml and deployment.xml for production deployment", + Long: `Modify service.xml and deployment.xml for production deployment. + +Only basic deployment configuration is available through this command. For +advanced configuration see the relevant Vespa Cloud documentation and make +changes to deployment.xml and services.xml directly. + +Reference: +https://cloud.vespa.ai/en/reference/services +https://cloud.vespa.ai/en/reference/deployment`, + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + pkg, err := cli.applicationPackageFrom(args, false) + if err != nil { + return err + } + if pkg.IsZip() { + return errHint(fmt.Errorf("cannot modify compressed application package %s", pkg.Path), + "Try running 'mvn clean' and run this command again") + } + + deploymentXML, err := readDeploymentXML(pkg) + if err != nil { + return fmt.Errorf("could not read deployment.xml: %w", err) + } + servicesXML, err := readServicesXML(pkg) + if err != nil { + return fmt.Errorf("a services.xml declaring your cluster(s) must exist: %w", err) + } + target, err := cli.target(targetOptions{noCertificate: true}) + if err != nil { + return err + } + + fmt.Fprint(cli.Stdout, "This will modify any existing ", color.YellowString("deployment.xml"), " and ", color.YellowString("services.xml"), + "!\nBefore modification a backup of the original file will be created.\n\n") + fmt.Fprint(cli.Stdout, "A default value is suggested (shown inside brackets) based on\nthe files' existing contents. Press enter to use it.\n\n") + fmt.Fprint(cli.Stdout, "Abort the configuration at any time by pressing Ctrl-C. The\nfiles will remain untouched.\n\n") + fmt.Fprint(cli.Stdout, "See this guide for sizing a Vespa deployment:\n", color.GreenString("https://docs.vespa.ai/en/performance/sizing-search.html\n\n")) + r := bufio.NewReader(cli.Stdin) + deploymentXML, err = updateRegions(cli, r, deploymentXML, target.Deployment().System) + if err != nil { + return err + } + servicesXML, err = updateNodes(cli, r, servicesXML) + if err != nil { + return err + } + + fmt.Fprintln(cli.Stdout) + if err := writeWithBackup(cli.Stdout, pkg, "deployment.xml", deploymentXML.String()); err != nil { + return err + } + if err := writeWithBackup(cli.Stdout, pkg, "services.xml", servicesXML.String()); err != nil { + return err + } + return nil + }, + } +} + +func newProdSubmitCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "submit", + Short: "Submit your application for production deployment", + Long: `Submit your application for production deployment. + +This commands uploads your application package to Vespa Cloud and deploys it to +the production zones specified in deployment.xml. + +Nodes are allocated to your application according to resources specified in +services.xml. + +While submitting an application from a local development environment is +supported, it's strongly recommended that production deployments are performed +by a continuous build system. + +For more information about production deployments in Vespa Cloud see: +https://cloud.vespa.ai/en/getting-to-production +https://cloud.vespa.ai/en/automated-deployments`, + DisableAutoGenTag: true, + SilenceUsage: true, + Example: `$ mvn package # when adding custom Java components +$ vespa prod submit`, + RunE: func(cmd *cobra.Command, args []string) error { + target, err := cli.target(targetOptions{noCertificate: true}) + if err != nil { + return err + } + if target.Type() != vespa.TargetCloud { + // TODO: Add support for hosted + return fmt.Errorf("prod submit does not support %s target", target.Type()) + } + pkg, err := cli.applicationPackageFrom(args, true) + if err != nil { + return err + } + if !pkg.HasDeployment() { + return errHint(fmt.Errorf("no deployment.xml found"), "Try creating one with vespa prod init") + } + if err := verifyTests(cli, pkg); err != nil { + return err + } + if !cli.isCI() { + cli.printWarning("We recommend doing this only from a CD job", "See https://cloud.vespa.ai/en/getting-to-production") + } + opts, err := cli.createDeploymentOptions(pkg, target) + if err != nil { + return err + } + if err := vespa.Submit(opts); err != nil { + return fmt.Errorf("could not submit application for deployment: %w", err) + } else { + cli.printSuccess("Submitted ", color.CyanString(pkg.Path), " for deployment") + log.Printf("See %s for deployment progress\n", color.CyanString(fmt.Sprintf("%s/tenant/%s/application/%s/prod/deployment", + opts.Target.Deployment().System.ConsoleURL, opts.Target.Deployment().Application.Tenant, opts.Target.Deployment().Application.Application))) + } + return nil + }, + } +} + +func writeWithBackup(stdout io.Writer, pkg vespa.ApplicationPackage, filename, contents string) error { + dst := filepath.Join(pkg.Path, filename) + if util.PathExists(dst) { + data, err := os.ReadFile(dst) + if err != nil { + return err + } + if bytes.Equal(data, []byte(contents)) { + fmt.Fprintf(stdout, "Not writing %s: File is unchanged\n", color.YellowString(filename)) + return nil + } + renamed := false + for i := 1; i <= 1000; i++ { + bak := fmt.Sprintf("%s.%d.bak", dst, i) + if !util.PathExists(bak) { + fmt.Fprintf(stdout, "Backing up existing %s to %s\n", color.YellowString(filename), color.YellowString(bak)) + if err := os.Rename(dst, bak); err != nil { + return err + } + renamed = true + break + } + } + if !renamed { + return fmt.Errorf("could not find an unused backup name for %s", dst) + } + } + fmt.Fprintf(stdout, "Writing %s\n", color.GreenString(dst)) + return os.WriteFile(dst, []byte(contents), 0644) +} + +func updateRegions(cli *CLI, stdin *bufio.Reader, deploymentXML xml.Deployment, system vespa.System) (xml.Deployment, error) { + regions, err := promptRegions(cli, stdin, deploymentXML, system) + if err != nil { + return xml.Deployment{}, err + } + parts := strings.Split(regions, ",") + regionElements := xml.Regions(parts...) + if err := deploymentXML.Replace("prod", "region", regionElements); err != nil { + return xml.Deployment{}, fmt.Errorf("could not update region elements in deployment.xml: %w", err) + } + // TODO: Some sample apps come with production <test> elements, but not necessarily working production tests, we + // therefore remove <test> elements here. + // This can be improved by supporting <test> elements in xml package and allow specifying testing as part of + // region prompt, e.g. region1;test,region2 + if err := deploymentXML.Replace("prod", "test", nil); err != nil { + return xml.Deployment{}, fmt.Errorf("could not remove test elements in deployment.xml: %w", err) + } + return deploymentXML, nil +} + +func promptRegions(cli *CLI, stdin *bufio.Reader, deploymentXML xml.Deployment, system vespa.System) (string, error) { + fmt.Fprintln(cli.Stdout, color.CyanString("> Deployment regions")) + fmt.Fprintf(cli.Stdout, "Documentation: %s\n", color.GreenString("https://cloud.vespa.ai/en/reference/zones")) + fmt.Fprintf(cli.Stdout, "Example: %s\n\n", color.YellowString("aws-us-east-1c,aws-us-west-2a")) + var currentRegions []string + for _, r := range deploymentXML.Prod.Regions { + currentRegions = append(currentRegions, r.Name) + } + if len(deploymentXML.Instance) > 0 { + for _, r := range deploymentXML.Instance[0].Prod.Regions { + currentRegions = append(currentRegions, r.Name) + } + } + validator := func(input string) error { + regions := strings.Split(input, ",") + for _, r := range regions { + if !xml.IsProdRegion(r, system) { + return fmt.Errorf("invalid region %s", r) + } + } + return nil + } + return prompt(cli, stdin, "Which regions do you wish to deploy in?", strings.Join(currentRegions, ","), validator) +} + +func updateNodes(cli *CLI, r *bufio.Reader, servicesXML xml.Services) (xml.Services, error) { + for _, c := range servicesXML.Container { + nodes, err := promptNodes(cli, r, c.ID, c.Nodes) + if err != nil { + return xml.Services{}, err + } + if err := servicesXML.Replace("container#"+c.ID, "nodes", nodes); err != nil { + return xml.Services{}, err + } + } + for _, c := range servicesXML.Content { + nodes, err := promptNodes(cli, r, c.ID, c.Nodes) + if err != nil { + return xml.Services{}, err + } + if err := servicesXML.Replace("content#"+c.ID, "nodes", nodes); err != nil { + return xml.Services{}, err + } + } + return servicesXML, nil +} + +func promptNodes(cli *CLI, r *bufio.Reader, clusterID string, defaultValue xml.Nodes) (xml.Nodes, error) { + count, err := promptNodeCount(cli, r, clusterID, defaultValue.Count) + if err != nil { + return xml.Nodes{}, err + } + const autoSpec = "auto" + defaultSpec := autoSpec + resources := defaultValue.Resources + if resources != nil { + defaultSpec = defaultValue.Resources.String() + } + spec, err := promptResources(cli, r, clusterID, defaultSpec) + if err != nil { + return xml.Nodes{}, err + } + if spec == autoSpec { + resources = nil + } else { + r, err := xml.ParseResources(spec) + if err != nil { + return xml.Nodes{}, err // Should not happen as resources have already been validated + } + resources = &r + } + return xml.Nodes{Count: count, Resources: resources}, nil +} + +func promptNodeCount(cli *CLI, stdin *bufio.Reader, clusterID string, nodeCount string) (string, error) { + fmt.Fprintln(cli.Stdout, color.CyanString("\n> Node count: "+clusterID+" cluster")) + fmt.Fprintf(cli.Stdout, "Documentation: %s\n", color.GreenString("https://cloud.vespa.ai/en/reference/services")) + fmt.Fprintf(cli.Stdout, "Example: %s\nExample: %s\n\n", color.YellowString("4"), color.YellowString("[2,8]")) + validator := func(input string) error { + _, _, err := xml.ParseNodeCount(input) + return err + } + return prompt(cli, stdin, fmt.Sprintf("How many nodes should the %s cluster have?", color.CyanString(clusterID)), nodeCount, validator) +} + +func promptResources(cli *CLI, stdin *bufio.Reader, clusterID string, resources string) (string, error) { + fmt.Fprintln(cli.Stdout, color.CyanString("\n> Node resources: "+clusterID+" cluster")) + fmt.Fprintf(cli.Stdout, "Documentation: %s\n", color.GreenString("https://cloud.vespa.ai/en/reference/services")) + fmt.Fprintf(cli.Stdout, "Example: %s\nExample: %s\n\n", color.YellowString("auto"), color.YellowString("vcpu=4,memory=8Gb,disk=100Gb")) + validator := func(input string) error { + if input == "auto" { + return nil + } + _, err := xml.ParseResources(input) + return err + } + return prompt(cli, stdin, fmt.Sprintf("Which resources should each node in the %s cluster have?", color.CyanString(clusterID)), resources, validator) +} + +func readDeploymentXML(pkg vespa.ApplicationPackage) (xml.Deployment, error) { + f, err := os.Open(filepath.Join(pkg.Path, "deployment.xml")) + if errors.Is(err, os.ErrNotExist) { + // Return a default value if there is no current deployment.xml + return xml.DefaultDeployment, nil + } else if err != nil { + return xml.Deployment{}, err + } + defer f.Close() + return xml.ReadDeployment(f) +} + +func readServicesXML(pkg vespa.ApplicationPackage) (xml.Services, error) { + f, err := os.Open(filepath.Join(pkg.Path, "services.xml")) + if err != nil { + return xml.Services{}, err + } + defer f.Close() + return xml.ReadServices(f) +} + +func prompt(cli *CLI, stdin *bufio.Reader, question, defaultAnswer string, validator func(input string) error) (string, error) { + var input string + for input == "" { + fmt.Fprint(cli.Stdout, question) + if defaultAnswer != "" { + fmt.Fprint(cli.Stdout, " [", color.YellowString(defaultAnswer), "]") + } + fmt.Fprint(cli.Stdout, " ") + + var err error + input, err = stdin.ReadString('\n') + if err != nil { + return "", err + } + input = strings.TrimSpace(input) + if input == "" { + input = defaultAnswer + } + + if err := validator(input); err != nil { + cli.printErr(err) + fmt.Fprintln(cli.Stderr) + input = "" + } + } + return input, nil +} + +func verifyTests(cli *CLI, app vespa.ApplicationPackage) error { + if !app.HasTests() { + return nil + } + // TODO: system-test, staging-setup and staging-test should be required if the application + // does not have any Java tests. + suites := map[string]bool{ + "system-test": false, + "staging-setup": false, + "staging-test": false, + "production-test": false, + } + testPath := app.TestPath + if app.IsZip() { + path, err := app.Unzip(true) + if err != nil { + return err + } + defer os.RemoveAll(path) + testPath = path + } + for suite, required := range suites { + if err := verifyTest(cli, testPath, suite, required); err != nil { + return err + } + } + return nil +} + +func verifyTest(cli *CLI, testsParent string, suite string, required bool) error { + testDirectory := filepath.Join(testsParent, "tests", suite) + _, err := os.Stat(testDirectory) + if err != nil { + if required { + if errors.Is(err, os.ErrNotExist) { + return errHint(fmt.Errorf("no %s tests found: %w", suite, err), + fmt.Sprintf("No such directory: %s", testDirectory), + "See https://cloud.vespa.ai/en/reference/testing") + } + return errHint(err, "See https://cloud.vespa.ai/en/reference/testing") + } + return nil + } + _, _, err = runTests(cli, testDirectory, true) + return err +} diff --git a/client/go/internal/cli/cmd/prod_test.go b/client/go/internal/cli/cmd/prod_test.go new file mode 100644 index 00000000000..b04e7861080 --- /dev/null +++ b/client/go/internal/cli/cmd/prod_test.go @@ -0,0 +1,271 @@ +package cmd + +import ( + "bytes" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vespa-engine/vespa/client/go/internal/mock" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func TestProdInit(t *testing.T) { + pkgDir := filepath.Join(t.TempDir(), "app") + createApplication(t, pkgDir, false, false) + + answers := []string{ + // Regions + "invalid input", + "aws-us-west-2a,aws-eu-west-1a", + + // Node count: qrs + "invalid input", + "4", + + // Node resources: qrs + "invalid input", + "auto", + + // Node count: music + "invalid input", + "6", + + // Node resources: music + "invalid input", + "vcpu=16,memory=64Gb,disk=100Gb", + } + var buf bytes.Buffer + buf.WriteString(strings.Join(answers, "\n") + "\n") + + cli, _, _ := newTestCLI(t) + cli.Stdin = &buf + assert.Nil(t, cli.Run("prod", "init", pkgDir)) + + // Verify contents + deploymentPath := filepath.Join(pkgDir, "src", "main", "application", "deployment.xml") + deploymentXML := readFileString(t, deploymentPath) + assert.Contains(t, deploymentXML, `<region>aws-us-west-2a</region>`) + assert.Contains(t, deploymentXML, `<region>aws-eu-west-1a</region>`) + + servicesPath := filepath.Join(pkgDir, "src", "main", "application", "services.xml") + servicesXML := readFileString(t, servicesPath) + containerFragment := `<container id="qrs" version="1.0"> + <document-api></document-api> + <search></search> + <nodes count="4"></nodes> + </container>` + assert.Contains(t, servicesXML, containerFragment) + contentFragment := `<content id="music" version="1.0"> + <redundancy>2</redundancy> + <documents> + <document type="music" mode="index"></document> + </documents> + <nodes count="6"> + <resources vcpu="16" memory="64Gb" disk="100Gb"></resources> + </nodes> + </content>` + assert.Contains(t, servicesXML, contentFragment) + + // Backups are created + assert.True(t, util.PathExists(deploymentPath+".1.bak")) + assert.True(t, util.PathExists(servicesPath+".1.bak")) +} + +func readFileString(t *testing.T, filename string) string { + content, err := os.ReadFile(filename) + if err != nil { + t.Fatal(err) + } + return string(content) +} + +func createApplication(t *testing.T, pkgDir string, java bool, skipTests bool) { + appDir := filepath.Join(pkgDir, "src", "main", "application") + targetDir := filepath.Join(pkgDir, "target") + if err := os.MkdirAll(appDir, 0755); err != nil { + t.Fatal(err) + } + + deploymentXML := `<deployment version="1.0"> + <prod> + <region>aws-us-east-1c</region> + </prod> +</deployment>` + if err := os.WriteFile(filepath.Join(appDir, "deployment.xml"), []byte(deploymentXML), 0644); err != nil { + t.Fatal(err) + } + + servicesXML := `<services version="1.0" xmlns:deploy="vespa" xmlns:preprocess="properties"> + <container id="qrs" version="1.0"> + <document-api/> + <search/> + <nodes count="2"> + <resources vcpu="4" memory="8Gb" disk="100Gb"/> + </nodes> + </container> + <content id="music" version="1.0"> + <redundancy>2</redundancy> + <documents> + <document type="music" mode="index"></document> + </documents> + <nodes count="4"></nodes> + </content> +</services>` + + if err := os.WriteFile(filepath.Join(appDir, "services.xml"), []byte(servicesXML), 0644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(targetDir, 0755); err != nil { + t.Fatal(err) + } + if java { + if skipTests { + t.Fatalf("skipTests=%t has no effect when java=%t", skipTests, java) + } + if err := os.WriteFile(filepath.Join(pkgDir, "pom.xml"), []byte(""), 0644); err != nil { + t.Fatal(err) + } + } else if !skipTests { + testsDir := filepath.Join(pkgDir, "src", "test", "application", "tests") + testBytes, _ := io.ReadAll(strings.NewReader("{\"steps\":[{}]}")) + writeTest(filepath.Join(testsDir, "system-test", "test.json"), testBytes, t) + writeTest(filepath.Join(testsDir, "staging-setup", "test.json"), testBytes, t) + writeTest(filepath.Join(testsDir, "staging-test", "test.json"), testBytes, t) + } +} + +func writeTest(path string, content []byte, t *testing.T) { + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, content, 0644); err != nil { + t.Fatal(err) + } +} + +func TestProdSubmit(t *testing.T) { + pkgDir := filepath.Join(t.TempDir(), "app") + createApplication(t, pkgDir, false, false) + prodSubmit(pkgDir, t) +} + +func TestProdSubmitWithoutTests(t *testing.T) { + pkgDir := filepath.Join(t.TempDir(), "app") + createApplication(t, pkgDir, false, true) + prodSubmit(pkgDir, t) +} + +func prodSubmit(pkgDir string, t *testing.T) { + t.Helper() + httpClient := &mock.HTTPClient{} + httpClient.NextResponseString(200, `ok`) + + cli, stdout, _ := newTestCLI(t, "CI=true") + cli.httpClient = httpClient + app := vespa.ApplicationID{Tenant: "t1", Application: "a1", Instance: "i1"} + assert.Nil(t, cli.Run("config", "set", "application", app.String())) + assert.Nil(t, cli.Run("config", "set", "target", "cloud")) + assert.Nil(t, cli.Run("auth", "api-key")) + assert.Nil(t, cli.Run("auth", "cert", pkgDir)) + + // Remove certificate as it's not required for submission (but it must be part of the application package) + if path, err := cli.config.privateKeyPath(app, vespa.TargetCloud); err == nil { + os.RemoveAll(path) + } else { + require.Nil(t, err) + } + if path, err := cli.config.certificatePath(app, vespa.TargetCloud); err == nil { + os.RemoveAll(path) + } else { + require.Nil(t, err) + } + + // Zipping requires relative paths, so must let command run from pkgDir, then reset cwd for subsequent tests. + if cwd, err := os.Getwd(); err != nil { + t.Fatal(err) + } else { + defer os.Chdir(cwd) + } + if err := os.Chdir(pkgDir); err != nil { + t.Fatal(err) + } + + stdout.Reset() + cli.Environment["VESPA_CLI_API_KEY_FILE"] = filepath.Join(cli.config.homeDir, "t1.api-key.pem") + assert.Nil(t, cli.Run("prod", "submit")) + assert.Contains(t, stdout.String(), "Success: Submitted") + assert.Contains(t, stdout.String(), "See https://console.vespa-cloud.com/tenant/t1/application/a1/prod/deployment for deployment progress") +} + +func TestProdSubmitWithJava(t *testing.T) { + pkgDir := filepath.Join(t.TempDir(), "app") + createApplication(t, pkgDir, true, false) + + httpClient := &mock.HTTPClient{} + httpClient.NextResponseString(200, `ok`) + cli, stdout, _ := newTestCLI(t, "CI=true") + cli.httpClient = httpClient + assert.Nil(t, cli.Run("config", "set", "application", "t1.a1.i1")) + assert.Nil(t, cli.Run("config", "set", "target", "cloud")) + assert.Nil(t, cli.Run("auth", "api-key")) + assert.Nil(t, cli.Run("auth", "cert", pkgDir)) + + // Copy an application package pre-assembled with mvn package + testAppDir := filepath.Join("testdata", "applications", "withDeployment", "target") + zipFile := filepath.Join(testAppDir, "application.zip") + copyFile(t, filepath.Join(pkgDir, "target", "application.zip"), zipFile) + testZipFile := filepath.Join(testAppDir, "application-test.zip") + copyFile(t, filepath.Join(pkgDir, "target", "application-test.zip"), testZipFile) + + stdout.Reset() + cli.Environment["VESPA_CLI_API_KEY_FILE"] = filepath.Join(cli.config.homeDir, "t1.api-key.pem") + assert.Nil(t, cli.Run("prod", "submit", pkgDir)) + assert.Contains(t, stdout.String(), "Success: Submitted") + assert.Contains(t, stdout.String(), "See https://console.vespa-cloud.com/tenant/t1/application/a1/prod/deployment for deployment progress") +} + +func TestProdSubmitInvalidZip(t *testing.T) { + pkgDir := filepath.Join(t.TempDir(), "app") + createApplication(t, pkgDir, true, false) + + httpClient := &mock.HTTPClient{} + httpClient.NextResponseString(200, `ok`) + cli, _, stderr := newTestCLI(t, "CI=true") + cli.httpClient = httpClient + assert.Nil(t, cli.Run("config", "set", "application", "t1.a1.i1")) + assert.Nil(t, cli.Run("config", "set", "target", "cloud")) + assert.Nil(t, cli.Run("auth", "api-key")) + assert.Nil(t, cli.Run("auth", "cert", pkgDir)) + + // Copy an invalid application package containing relative file names + testAppDir := filepath.Join("testdata", "applications", "withInvalidEntries", "target") + zipFile := filepath.Join(testAppDir, "application.zip") + copyFile(t, filepath.Join(pkgDir, "target", "application.zip"), zipFile) + testZipFile := filepath.Join(testAppDir, "application-test.zip") + copyFile(t, filepath.Join(pkgDir, "target", "application-test.zip"), testZipFile) + + assert.NotNil(t, cli.Run("prod", "submit", pkgDir)) + assert.Equal(t, "Error: found invalid path inside zip: ../../../../../../../tmp/foo\n", stderr.String()) +} + +func copyFile(t *testing.T, dstFilename, srcFilename string) { + dst, err := os.Create(dstFilename) + if err != nil { + t.Fatal(err) + } + defer dst.Close() + src, err := os.Open(srcFilename) + if err != nil { + t.Fatal(err) + } + defer src.Close() + if _, err := io.Copy(dst, src); err != nil { + t.Fatal(err) + } +} diff --git a/client/go/internal/cli/cmd/query.go b/client/go/internal/cli/cmd/query.go new file mode 100644 index 00000000000..a14e2d51036 --- /dev/null +++ b/client/go/internal/cli/cmd/query.go @@ -0,0 +1,117 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa query command +// author: bratseth + +package cmd + +import ( + "fmt" + "io" + "log" + "net/http" + "net/url" + "strings" + "time" + + "github.com/fatih/color" + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/curl" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func newQueryCmd(cli *CLI) *cobra.Command { + var ( + printCurl bool + queryTimeoutSecs int + ) + cmd := &cobra.Command{ + Use: "query query-parameters", + Short: "Issue a query to Vespa", + Example: `$ vespa query "yql=select * from music where album contains 'head';" hits=5`, + Long: `Issue a query to Vespa. + +Any parameter from https://docs.vespa.ai/en/reference/query-api-reference.html +can be set by the syntax [parameter-name]=[value].`, + // TODO: Support referencing a query json file + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return query(cli, args, queryTimeoutSecs, printCurl) + }, + } + cmd.PersistentFlags().BoolVarP(&printCurl, "verbose", "v", false, "Print the equivalent curl command for the query") + cmd.Flags().IntVarP(&queryTimeoutSecs, "timeout", "T", 10, "Timeout for the query in seconds") + return cmd +} + +func printCurl(stderr io.Writer, url string, service *vespa.Service) error { + cmd, err := curl.RawArgs(url) + if err != nil { + return err + } + cmd.Certificate = service.TLSOptions.CertificateFile + cmd.PrivateKey = service.TLSOptions.PrivateKeyFile + _, err = io.WriteString(stderr, cmd.String()+"\n") + return err +} + +func query(cli *CLI, arguments []string, timeoutSecs int, curl bool) error { + target, err := cli.target(targetOptions{}) + if err != nil { + return err + } + service, err := cli.service(target, vespa.QueryService, 0, cli.config.cluster()) + if err != nil { + return err + } + url, _ := url.Parse(service.BaseURL + "/search/") + urlQuery := url.Query() + for i := 0; i < len(arguments); i++ { + key, value := splitArg(arguments[i]) + urlQuery.Set(key, value) + } + queryTimeout := urlQuery.Get("timeout") + if queryTimeout == "" { + // No timeout set by user, use the timeout option + queryTimeout = fmt.Sprintf("%ds", timeoutSecs) + urlQuery.Set("timeout", queryTimeout) + } + url.RawQuery = urlQuery.Encode() + deadline, err := time.ParseDuration(queryTimeout) + if err != nil { + return fmt.Errorf("invalid query timeout: %w", err) + } + if curl { + if err := printCurl(cli.Stderr, url.String(), service); err != nil { + return err + } + } + response, err := service.Do(&http.Request{URL: url}, deadline+time.Second) // Slightly longer than query timeout + if err != nil { + return fmt.Errorf("request failed: %w", err) + } + defer response.Body.Close() + + if response.StatusCode == 200 { + log.Print(util.ReaderToJSON(response.Body)) + } else if response.StatusCode/100 == 4 { + return fmt.Errorf("invalid query: %s\n%s", response.Status, util.ReaderToJSON(response.Body)) + } else { + return fmt.Errorf("%s from container at %s\n%s", response.Status, color.CyanString(url.Host), util.ReaderToJSON(response.Body)) + } + return nil +} + +func splitArg(argument string) (string, string) { + parts := strings.SplitN(argument, "=", 2) + if len(parts) < 2 { + return "yql", parts[0] + } + if strings.HasPrefix(strings.ToLower(parts[0]), "select ") { + // A query containing '=' + return "yql", argument + } + return parts[0], parts[1] +} diff --git a/client/go/internal/cli/cmd/query_test.go b/client/go/internal/cli/cmd/query_test.go new file mode 100644 index 00000000000..94b5a485b9d --- /dev/null +++ b/client/go/internal/cli/cmd/query_test.go @@ -0,0 +1,115 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// query command tests +// Author: bratseth + +package cmd + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vespa-engine/vespa/client/go/internal/mock" +) + +func TestQuery(t *testing.T) { + assertQuery(t, + "?timeout=10s&yql=select+from+sources+%2A+where+title+contains+%27foo%27", + "select from sources * where title contains 'foo'") +} + +func TestQueryVerbose(t *testing.T) { + client := &mock.HTTPClient{} + client.NextResponseString(200, "{\"query\":\"result\"}") + + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = client + + assert.Nil(t, cli.Run("query", "-v", "select from sources * where title contains 'foo'")) + assert.Equal(t, "curl http://127.0.0.1:8080/search/\\?timeout=10s\\&yql=select+from+sources+%2A+where+title+contains+%27foo%27\n", stderr.String()) + assert.Equal(t, "{\n \"query\": \"result\"\n}\n", stdout.String()) +} + +func TestQueryNonJsonResult(t *testing.T) { + assertQuery(t, + "?timeout=10s&yql=select+from+sources+%2A+where+title+contains+%27foo%27", + "select from sources * where title contains 'foo'") +} + +func TestQueryWithMultipleParameters(t *testing.T) { + assertQuery(t, + "?hits=5&timeout=20s&yql=select+from+sources+%2A+where+title+contains+%27foo%27+and+year+%3D+2000", + "select from sources * where title contains 'foo' and year = 2000", "hits=5", "timeout=20s") +} + +func TestQueryWithEquals(t *testing.T) { + assertQuery(t, + "?timeout=10s&yql=SELECT+from+sources+%2A+where+title+contains+%27foo%27+and+year+%3D+2000", + "SELECT from sources * where title contains 'foo' and year = 2000") +} + +func TestQuerySelect(t *testing.T) { + assertQuery(t, + "?hits=5&select=%7B%22select%22%3A%7B%22where%22%3A%7B%22contains%22%3A%5B%22title%22%2C%22a%22%5D%7D%7D%7D&timeout=10s", + `select={"select":{"where":{"contains":["title","a"]}}}`, "hits=5") +} + +func TestQueryWithExplicitYqlParameter(t *testing.T) { + assertQuery(t, + "?timeout=10s&yql=select+from+sources+%2A+where+title+contains+%27foo%27", + "yql=select from sources * where title contains 'foo'") +} + +func TestIllegalQuery(t *testing.T) { + assertQueryError(t, 401, "query error message") +} + +func TestServerError(t *testing.T) { + assertQueryServiceError(t, 501, "server error message") +} + +func assertQuery(t *testing.T, expectedQuery string, query ...string) { + client := &mock.HTTPClient{} + client.NextResponseString(200, "{\"query\":\"result\"}") + cli, stdout, _ := newTestCLI(t) + cli.httpClient = client + + args := []string{"query"} + assert.Nil(t, cli.Run(append(args, query...)...)) + assert.Equal(t, + "{\n \"query\": \"result\"\n}\n", + stdout.String(), + "query output") + queryURL, err := queryServiceURL(client) + require.Nil(t, err) + assert.Equal(t, queryURL+"/search/"+expectedQuery, client.LastRequest.URL.String()) +} + +func assertQueryError(t *testing.T, status int, errorMessage string) { + client := &mock.HTTPClient{} + client.NextResponseString(status, errorMessage) + cli, _, stderr := newTestCLI(t) + cli.httpClient = client + assert.NotNil(t, cli.Run("query", "yql=select from sources * where title contains 'foo'")) + assert.Equal(t, + "Error: invalid query: Status "+strconv.Itoa(status)+"\n"+errorMessage+"\n", + stderr.String(), + "error output") +} + +func assertQueryServiceError(t *testing.T, status int, errorMessage string) { + client := &mock.HTTPClient{} + client.NextResponseString(status, errorMessage) + cli, _, stderr := newTestCLI(t) + cli.httpClient = client + assert.NotNil(t, cli.Run("query", "yql=select from sources * where title contains 'foo'")) + assert.Equal(t, + "Error: Status "+strconv.Itoa(status)+" from container at 127.0.0.1:8080\n"+errorMessage+"\n", + stderr.String(), + "error output") +} + +func queryServiceURL(client *mock.HTTPClient) (string, error) { + return "http://127.0.0.1:8080", nil +} diff --git a/client/go/internal/cli/cmd/root.go b/client/go/internal/cli/cmd/root.go new file mode 100644 index 00000000000..faba6bbbfd4 --- /dev/null +++ b/client/go/internal/cli/cmd/root.go @@ -0,0 +1,512 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package cmd + +import ( + "encoding/json" + "fmt" + "io" + "log" + "os" + "os/exec" + "strings" + "time" + + "github.com/fatih/color" + "github.com/mattn/go-colorable" + "github.com/mattn/go-isatty" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/vespa-engine/vespa/client/go/internal/cli/build" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/version" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +const ( + applicationFlag = "application" + instanceFlag = "instance" + clusterFlag = "cluster" + zoneFlag = "zone" + targetFlag = "target" + waitFlag = "wait" + colorFlag = "color" + quietFlag = "quiet" +) + +// CLI holds the Vespa CLI command tree, configuration and dependencies. +type CLI struct { + // Environment holds the process environment. + Environment map[string]string + Stdin io.ReadWriter + Stdout io.Writer + Stderr io.Writer + + cmd *cobra.Command + config *Config + version version.Version + + httpClient util.HTTPClient + exec executor + isTerminal func() bool + spinner func(w io.Writer, message string, fn func() error) error +} + +// ErrCLI is an error returned to the user. It wraps an exit status, a regular error and optional hints for resolving +// the error. +type ErrCLI struct { + Status int + quiet bool + hints []string + error +} + +type targetOptions struct { + // logLevel sets the log level to use for this target. If empty, it defaults to "info". + logLevel string + // noCertificate declares that no client certificate should be required when using this target. + noCertificate bool +} + +// errHint creates a new CLI error, with optional hints that will be printed after the error +func errHint(err error, hints ...string) ErrCLI { return ErrCLI{Status: 1, hints: hints, error: err} } + +type executor interface { + LookPath(name string) (string, error) + Run(name string, args ...string) ([]byte, error) +} + +type execSubprocess struct{} + +func (c *execSubprocess) LookPath(name string) (string, error) { return exec.LookPath(name) } +func (c *execSubprocess) Run(name string, args ...string) ([]byte, error) { + return exec.Command(name, args...).Output() +} + +// New creates the Vespa CLI, writing output to stdout and stderr, and reading environment variables from environment. +func New(stdout, stderr io.Writer, environment []string) (*CLI, error) { + cmd := &cobra.Command{ + Use: "vespa command-name", + Short: "The command-line tool for Vespa.ai", + Long: `The command-line tool for Vespa.ai. + +Use it on Vespa instances running locally, remotely or in the cloud. +Prefer web service API's to this in production. + +Vespa documentation: https://docs.vespa.ai + +For detailed description of flags and configuration, see 'vespa help config'. +`, + DisableAutoGenTag: true, + SilenceErrors: true, // We have our own error printing + SilenceUsage: false, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return fmt.Errorf("invalid command: %s", args[0]) + }, + } + env := make(map[string]string) + for _, entry := range environment { + parts := strings.SplitN(entry, "=", 2) + env[parts[0]] = parts[1] + } + version, err := version.Parse(build.Version) + if err != nil { + return nil, err + } + cli := CLI{ + Environment: env, + Stdin: os.Stdin, + Stdout: stdout, + Stderr: stderr, + + version: version, + cmd: cmd, + httpClient: util.CreateClient(time.Second * 10), + exec: &execSubprocess{}, + } + cli.isTerminal = func() bool { return isTerminal(cli.Stdout) && isTerminal(cli.Stderr) } + if err := cli.loadConfig(); err != nil { + return nil, err + } + cli.configureSpinner() + cli.configureCommands() + cmd.PersistentPreRunE = cli.configureOutput + return &cli, nil +} + +func (c *CLI) loadConfig() error { + config, err := loadConfig(c.Environment, c.configureFlags()) + if err != nil { + return err + } + c.config = config + return nil +} + +func (c *CLI) configureOutput(cmd *cobra.Command, args []string) error { + if f, ok := c.Stdout.(*os.File); ok { + c.Stdout = colorable.NewColorable(f) + } + if f, ok := c.Stderr.(*os.File); ok { + c.Stderr = colorable.NewColorable(f) + } + if c.config.isQuiet() { + c.Stdout = io.Discard + } + log.SetFlags(0) // No timestamps + log.SetOutput(c.Stdout) + colorValue, _ := c.config.get(colorFlag) + colorize := false + switch colorValue { + case "auto": + _, nocolor := c.Environment["NO_COLOR"] // https://no-color.org + colorize = !nocolor && c.isTerminal() + case "always": + colorize = true + case "never": + default: + return fmt.Errorf("invalid color option: %s", colorValue) + } + color.NoColor = !colorize + return nil +} + +func (c *CLI) configureFlags() map[string]*pflag.Flag { + var ( + target string + application string + instance string + cluster string + zone string + waitSecs int + color string + quiet bool + ) + c.cmd.PersistentFlags().StringVarP(&target, targetFlag, "t", "local", `The target platform to use. Must be "local", "cloud", "hosted" or an URL`) + c.cmd.PersistentFlags().StringVarP(&application, applicationFlag, "a", "", "The application to use") + c.cmd.PersistentFlags().StringVarP(&instance, instanceFlag, "i", "", "The instance of the application to use") + c.cmd.PersistentFlags().StringVarP(&cluster, clusterFlag, "C", "", "The container cluster to use. This is only required for applications with multiple clusters") + c.cmd.PersistentFlags().StringVarP(&zone, zoneFlag, "z", "", "The zone to use. This defaults to a dev zone") + c.cmd.PersistentFlags().IntVarP(&waitSecs, waitFlag, "w", 0, "Number of seconds to wait for a service to become ready") + c.cmd.PersistentFlags().StringVarP(&color, colorFlag, "c", "auto", `Whether to use colors in output. Must be "auto", "never", or "always"`) + c.cmd.PersistentFlags().BoolVarP(&quiet, quietFlag, "q", false, "Print only errors") + flags := make(map[string]*pflag.Flag) + c.cmd.PersistentFlags().VisitAll(func(flag *pflag.Flag) { + flags[flag.Name] = flag + }) + return flags +} + +func (c *CLI) configureSpinner() { + // Explicitly disable spinner for Screwdriver. It emulates a tty but + // \r result in a newline, and output gets truncated. + _, screwdriver := c.Environment["SCREWDRIVER"] + if c.config.isQuiet() || !c.isTerminal() || screwdriver { + c.spinner = func(w io.Writer, message string, fn func() error) error { + return fn() + } + } else { + c.spinner = util.Spinner + } +} + +func (c *CLI) configureCommands() { + rootCmd := c.cmd + authCmd := newAuthCmd() + certCmd := newCertCmd(c) + configCmd := newConfigCmd() + documentCmd := newDocumentCmd(c) + prodCmd := newProdCmd() + statusCmd := newStatusCmd(c) + certCmd.AddCommand(newCertAddCmd(c)) // auth cert add + authCmd.AddCommand(certCmd) // auth cert + authCmd.AddCommand(newAPIKeyCmd(c)) // auth api-key + authCmd.AddCommand(newLoginCmd(c)) // auth login + authCmd.AddCommand(newLogoutCmd(c)) // auth logout + rootCmd.AddCommand(authCmd) // auth + rootCmd.AddCommand(newCloneCmd(c)) // clone + configCmd.AddCommand(newConfigGetCmd(c)) // config get + configCmd.AddCommand(newConfigSetCmd(c)) // config set + configCmd.AddCommand(newConfigUnsetCmd(c)) // config unset + rootCmd.AddCommand(configCmd) // config + rootCmd.AddCommand(newCurlCmd(c)) // curl + rootCmd.AddCommand(newDeployCmd(c)) // deploy + rootCmd.AddCommand(newPrepareCmd(c)) // prepare + rootCmd.AddCommand(newActivateCmd(c)) // activate + documentCmd.AddCommand(newDocumentPutCmd(c)) // document put + documentCmd.AddCommand(newDocumentUpdateCmd(c)) // document update + documentCmd.AddCommand(newDocumentRemoveCmd(c)) // document remove + documentCmd.AddCommand(newDocumentGetCmd(c)) // document get + rootCmd.AddCommand(documentCmd) // document + rootCmd.AddCommand(newLogCmd(c)) // log + rootCmd.AddCommand(newManCmd(c)) // man + prodCmd.AddCommand(newProdInitCmd(c)) // prod init + prodCmd.AddCommand(newProdSubmitCmd(c)) // prod submit + rootCmd.AddCommand(prodCmd) // prod + rootCmd.AddCommand(newQueryCmd(c)) // query + statusCmd.AddCommand(newStatusQueryCmd(c)) // status query + statusCmd.AddCommand(newStatusDocumentCmd(c)) // status document + statusCmd.AddCommand(newStatusDeployCmd(c)) // status deploy + rootCmd.AddCommand(statusCmd) // status + rootCmd.AddCommand(newTestCmd(c)) // test + rootCmd.AddCommand(newVersionCmd(c)) // version +} + +func (c *CLI) printErr(err error, hints ...string) { + fmt.Fprintln(c.Stderr, color.RedString("Error:"), err) + for _, hint := range hints { + fmt.Fprintln(c.Stderr, color.CyanString("Hint:"), hint) + } +} + +func (c *CLI) printSuccess(msg ...interface{}) { + fmt.Fprintln(c.Stdout, color.GreenString("Success:"), fmt.Sprint(msg...)) +} + +func (c *CLI) printWarning(msg interface{}, hints ...string) { + fmt.Fprintln(c.Stderr, color.YellowString("Warning:"), msg) + for _, hint := range hints { + fmt.Fprintln(c.Stderr, color.CyanString("Hint:"), hint) + } +} + +// target creates a target according the configuration of this CLI and given opts. +func (c *CLI) target(opts targetOptions) (vespa.Target, error) { + target, err := c.createTarget(opts) + if err != nil { + return nil, err + } + if !c.isCloudCI() { // Vespa Cloud always runs an up-to-date version + if err := target.CheckVersion(c.version); err != nil { + c.printWarning(err, "This version may not work as expected", "Try 'vespa version' to check for a new version") + } + } + return target, nil +} + +func (c *CLI) createTarget(opts targetOptions) (vespa.Target, error) { + targetType, err := c.config.targetType() + if err != nil { + return nil, err + } + if strings.HasPrefix(targetType, "http") { + return vespa.CustomTarget(c.httpClient, targetType), nil + } + switch targetType { + case vespa.TargetLocal: + return vespa.LocalTarget(c.httpClient), nil + case vespa.TargetCloud, vespa.TargetHosted: + return c.createCloudTarget(targetType, opts) + } + return nil, errHint(fmt.Errorf("invalid target: %s", targetType), "Valid targets are 'local', 'cloud', 'hosted' or an URL") +} + +func (c *CLI) createCloudTarget(targetType string, opts targetOptions) (vespa.Target, error) { + system, err := c.system(targetType) + if err != nil { + return nil, err + } + deployment, err := c.config.deploymentIn(system) + if err != nil { + return nil, err + } + endpoints, err := c.endpointsFromEnv() + if err != nil { + return nil, err + } + var ( + apiKey []byte + authConfigPath string + apiTLSOptions vespa.TLSOptions + deploymentTLSOptions vespa.TLSOptions + ) + switch targetType { + case vespa.TargetCloud: + apiKey, err = c.config.readAPIKey(c, system, deployment.Application.Tenant) + if err != nil { + return nil, err + } + authConfigPath = c.config.authConfigPath() + deploymentTLSOptions = vespa.TLSOptions{} + if !opts.noCertificate { + kp, err := c.config.x509KeyPair(deployment.Application, targetType) + if err != nil { + return nil, errHint(err, "Deployment to cloud requires a certificate. Try 'vespa auth cert'") + } + deploymentTLSOptions = vespa.TLSOptions{ + KeyPair: kp.KeyPair, + CertificateFile: kp.CertificateFile, + PrivateKeyFile: kp.PrivateKeyFile, + } + } + case vespa.TargetHosted: + kp, err := c.config.x509KeyPair(deployment.Application, targetType) + if err != nil { + return nil, errHint(err, "Deployment to hosted requires an Athenz certificate", "Try renewing certificate with 'athenz-user-cert'") + } + apiTLSOptions = vespa.TLSOptions{ + KeyPair: kp.KeyPair, + CertificateFile: kp.CertificateFile, + PrivateKeyFile: kp.PrivateKeyFile, + } + deploymentTLSOptions = apiTLSOptions + default: + return nil, fmt.Errorf("invalid cloud target: %s", targetType) + } + apiOptions := vespa.APIOptions{ + System: system, + TLSOptions: apiTLSOptions, + APIKey: apiKey, + AuthConfigPath: authConfigPath, + } + deploymentOptions := vespa.CloudDeploymentOptions{ + Deployment: deployment, + TLSOptions: deploymentTLSOptions, + ClusterURLs: endpoints, + } + logLevel := opts.logLevel + if logLevel == "" { + logLevel = "info" + } + logOptions := vespa.LogOptions{ + Writer: c.Stdout, + Level: vespa.LogLevel(logLevel), + } + return vespa.CloudTarget(c.httpClient, apiOptions, deploymentOptions, logOptions) +} + +// system returns the appropiate system for the target configured in this CLI. +func (c *CLI) system(targetType string) (vespa.System, error) { + name := c.Environment["VESPA_CLI_CLOUD_SYSTEM"] + if name != "" { + return vespa.GetSystem(name) + } + switch targetType { + case vespa.TargetHosted: + return vespa.MainSystem, nil + case vespa.TargetCloud: + return vespa.PublicSystem, nil + } + return vespa.System{}, fmt.Errorf("no default system found for %s target", targetType) +} + +// service returns the service of given name located at target. If non-empty, cluster specifies a cluster to query. This +// function blocks according to the wait period configured in this CLI. The parameter sessionOrRunID specifies either +// the session ID (local target) or run ID (cloud target) to wait for. +func (c *CLI) service(target vespa.Target, name string, sessionOrRunID int64, cluster string) (*vespa.Service, error) { + timeout, err := c.config.timeout() + if err != nil { + return nil, err + } + if timeout > 0 { + log.Printf("Waiting up to %s for %s service to become available ...", color.CyanString(timeout.String()), color.CyanString(name)) + } + s, err := target.Service(name, timeout, sessionOrRunID, cluster) + if err != nil { + err := fmt.Errorf("service '%s' is unavailable: %w", name, err) + if target.IsCloud() { + return nil, errHint(err, "Confirm that you're communicating with the correct zone and cluster", "The -z option controls the zone", "The -C option controls the cluster") + } + return nil, err + } + return s, nil +} + +func (c *CLI) createDeploymentOptions(pkg vespa.ApplicationPackage, target vespa.Target) (vespa.DeploymentOptions, error) { + timeout, err := c.config.timeout() + if err != nil { + return vespa.DeploymentOptions{}, err + } + return vespa.DeploymentOptions{ + ApplicationPackage: pkg, + Target: target, + Timeout: timeout, + HTTPClient: c.httpClient, + }, nil +} + +// isCI returns true if running inside a continuous integration environment. +func (c *CLI) isCI() bool { + _, ok := c.Environment["CI"] + return ok +} + +// isCloudCI returns true if running inside a Vespa Cloud deployment job. +func (c *CLI) isCloudCI() bool { + _, ok := c.Environment["VESPA_CLI_CLOUD_CI"] + return ok +} + +func (c *CLI) endpointsFromEnv() (map[string]string, error) { + endpointsString := c.Environment["VESPA_CLI_ENDPOINTS"] + if endpointsString == "" { + return nil, nil + } + var endpoints endpoints + urlsByCluster := make(map[string]string) + if err := json.Unmarshal([]byte(endpointsString), &endpoints); err != nil { + return nil, fmt.Errorf("endpoints must be valid json: %w", err) + } + if len(endpoints.Endpoints) == 0 { + return nil, fmt.Errorf("endpoints must be non-empty") + } + for _, endpoint := range endpoints.Endpoints { + urlsByCluster[endpoint.Cluster] = endpoint.URL + } + return urlsByCluster, nil +} + +// Run executes the CLI with given args. If args is nil, it defaults to os.Args[1:]. +func (c *CLI) Run(args ...string) error { + c.cmd.SetArgs(args) + err := c.cmd.Execute() + if err != nil { + if cliErr, ok := err.(ErrCLI); ok { + if !cliErr.quiet { + c.printErr(cliErr, cliErr.hints...) + } + } else { + c.printErr(err) + } + } + return err +} + +type endpoints struct { + Endpoints []endpoint `json:"endpoints"` +} + +type endpoint struct { + Cluster string `json:"cluster"` + URL string `json:"url"` +} + +func isTerminal(w io.Writer) bool { + if f, ok := w.(*os.File); ok { + return isatty.IsTerminal(f.Fd()) + } + return false +} + +// applicationPackageFrom returns an application loaded from args. If args is empty, the application package is loaded +// from the working directory. If requirePackaging is true, the application package is required to be packaged with mvn +// package. +func (c *CLI) applicationPackageFrom(args []string, requirePackaging bool) (vespa.ApplicationPackage, error) { + path := "." + if len(args) == 1 { + path = args[0] + stat, err := os.Stat(path) + if err != nil { + return vespa.ApplicationPackage{}, err + } + if stat.IsDir() { + // Using an explicit application directory, look for local config in that directory too + if err := c.config.loadLocalConfigFrom(path); err != nil { + return vespa.ApplicationPackage{}, err + } + } + } else if len(args) > 1 { + return vespa.ApplicationPackage{}, fmt.Errorf("expected 0 or 1 arguments, got %d", len(args)) + } + return vespa.FindApplicationPackage(path, requirePackaging) +} diff --git a/client/go/internal/cli/cmd/status.go b/client/go/internal/cli/cmd/status.go new file mode 100644 index 00000000000..ab98a4da160 --- /dev/null +++ b/client/go/internal/cli/cmd/status.go @@ -0,0 +1,100 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa status command +// author: bratseth + +package cmd + +import ( + "fmt" + "log" + + "github.com/fatih/color" + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func newStatusCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "status", + Short: "Verify that a service is ready to use (query by default)", + Example: `$ vespa status query`, + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return printServiceStatus(cli, vespa.QueryService) + }, + } +} + +func newStatusQueryCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "query", + Short: "Verify that the query service is ready to use (default)", + Example: `$ vespa status query`, + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + return printServiceStatus(cli, vespa.QueryService) + }, + } +} + +func newStatusDocumentCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "document", + Short: "Verify that the document service is ready to use", + Example: `$ vespa status document`, + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + return printServiceStatus(cli, vespa.DocumentService) + }, + } +} + +func newStatusDeployCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "deploy", + Short: "Verify that the deploy service is ready to use", + Example: `$ vespa status deploy`, + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + return printServiceStatus(cli, vespa.DeployService) + }, + } +} + +func printServiceStatus(cli *CLI, name string) error { + t, err := cli.target(targetOptions{}) + if err != nil { + return err + } + cluster := cli.config.cluster() + s, err := cli.service(t, name, 0, cluster) + if err != nil { + return err + } + timeout, err := cli.config.timeout() + if err != nil { + return err + } + status, err := s.Wait(timeout) + clusterPart := "" + if cluster != "" { + clusterPart = fmt.Sprintf(" named %s", color.CyanString(cluster)) + } + if status/100 == 2 { + log.Print(s.Description(), clusterPart, " at ", color.CyanString(s.BaseURL), " is ", color.GreenString("ready")) + } else { + if err == nil { + err = fmt.Errorf("status %d", status) + } + return fmt.Errorf("%s%s at %s is %s: %w", s.Description(), clusterPart, color.CyanString(s.BaseURL), color.RedString("not ready"), err) + } + return nil +} diff --git a/client/go/internal/cli/cmd/status_test.go b/client/go/internal/cli/cmd/status_test.go new file mode 100644 index 00000000000..a3cae7c3fe4 --- /dev/null +++ b/client/go/internal/cli/cmd/status_test.go @@ -0,0 +1,103 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// status command tests +// Author: bratseth + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/mock" +) + +func TestStatusDeployCommand(t *testing.T) { + assertDeployStatus("http://127.0.0.1:19071", []string{}, t) +} + +func TestStatusDeployCommandWithURLTarget(t *testing.T) { + assertDeployStatus("http://mydeploytarget:19071", []string{"-t", "http://mydeploytarget"}, t) +} + +func TestStatusDeployCommandWithLocalTarget(t *testing.T) { + assertDeployStatus("http://127.0.0.1:19071", []string{"-t", "local"}, t) +} + +func TestStatusQueryCommand(t *testing.T) { + assertQueryStatus("http://127.0.0.1:8080", []string{}, t) +} + +func TestStatusQueryCommandWithUrlTarget(t *testing.T) { + assertQueryStatus("http://mycontainertarget:8080", []string{"-t", "http://mycontainertarget"}, t) +} + +func TestStatusQueryCommandWithLocalTarget(t *testing.T) { + assertQueryStatus("http://127.0.0.1:8080", []string{"-t", "local"}, t) +} + +func TestStatusDocumentCommandWithLocalTarget(t *testing.T) { + assertDocumentStatus("http://127.0.0.1:8080", []string{"-t", "local"}, t) +} + +func TestStatusErrorResponse(t *testing.T) { + assertQueryStatusError("http://127.0.0.1:8080", []string{}, t) +} + +func assertDeployStatus(target string, args []string, t *testing.T) { + client := &mock.HTTPClient{} + cli, stdout, _ := newTestCLI(t) + cli.httpClient = client + statusArgs := []string{"status", "deploy"} + assert.Nil(t, cli.Run(append(statusArgs, args...)...)) + assert.Equal(t, + "Deploy API at "+target+" is ready\n", + stdout.String(), + "vespa status config-server") + assert.Equal(t, target+"/status.html", client.LastRequest.URL.String()) +} + +func assertQueryStatus(target string, args []string, t *testing.T) { + client := &mock.HTTPClient{} + cli, stdout, _ := newTestCLI(t) + cli.httpClient = client + statusArgs := []string{"status", "query"} + assert.Nil(t, cli.Run(append(statusArgs, args...)...)) + assert.Equal(t, + "Container (query API) at "+target+" is ready\n", + stdout.String(), + "vespa status container") + assert.Equal(t, target+"/ApplicationStatus", client.LastRequest.URL.String()) + + statusArgs = []string{"status"} + stdout.Reset() + assert.Nil(t, cli.Run(append(statusArgs, args...)...)) + assert.Equal(t, + "Container (query API) at "+target+" is ready\n", + stdout.String(), + "vespa status (the default)") + assert.Equal(t, target+"/ApplicationStatus", client.LastRequest.URL.String()) +} + +func assertDocumentStatus(target string, args []string, t *testing.T) { + client := &mock.HTTPClient{} + cli, stdout, _ := newTestCLI(t) + cli.httpClient = client + assert.Nil(t, cli.Run("status", "document")) + assert.Equal(t, + "Container (document API) at "+target+" is ready\n", + stdout.String(), + "vespa status container") + assert.Equal(t, target+"/ApplicationStatus", client.LastRequest.URL.String()) +} + +func assertQueryStatusError(target string, args []string, t *testing.T) { + client := &mock.HTTPClient{} + client.NextStatus(500) + cli, _, stderr := newTestCLI(t) + cli.httpClient = client + assert.NotNil(t, cli.Run("status", "container")) + assert.Equal(t, + "Error: Container (query API) at "+target+" is not ready: status 500\n", + stderr.String(), + "vespa status container") +} diff --git a/client/go/internal/cli/cmd/test.go b/client/go/internal/cli/cmd/test.go new file mode 100644 index 00000000000..d071f9556a2 --- /dev/null +++ b/client/go/internal/cli/cmd/test.go @@ -0,0 +1,486 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa test command +// Author: jonmv + +package cmd + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "math" + "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/fatih/color" + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func newTestCmd(cli *CLI) *cobra.Command { + testCmd := &cobra.Command{ + Use: "test test-directory-or-file", + Short: "Run a test suite, or a single test", + Long: `Run a test suite, or a single test + +Runs all JSON test files in the specified directory, or the single JSON test file specified. + +See https://docs.vespa.ai/en/reference/testing.html for details.`, + Example: `$ vespa test src/test/application/tests/system-test +$ vespa test src/test/application/tests/system-test/feed-and-query.json`, + Args: cobra.ExactArgs(1), + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + count, failed, err := runTests(cli, args[0], false) + if err != nil { + return err + } + if len(failed) != 0 { + plural := "s" + if count == 1 { + plural = "" + } + fmt.Fprintf(cli.Stdout, "\n%s %d of %d test%s failed:\n", color.RedString("Failure:"), len(failed), count, plural) + for _, test := range failed { + fmt.Fprintln(cli.Stdout, test) + } + return ErrCLI{Status: 3, error: fmt.Errorf("tests failed"), quiet: true} + } else { + plural := "s" + if count == 1 { + plural = "" + } + fmt.Fprintf(cli.Stdout, "\n%s %d test%s OK\n", color.GreenString("Success:"), count, plural) + return nil + } + }, + } + return testCmd +} + +func runTests(cli *CLI, rootPath string, dryRun bool) (int, []string, error) { + count := 0 + failed := make([]string, 0) + if stat, err := os.Stat(rootPath); err != nil { + return 0, nil, errHint(err, "See https://docs.vespa.ai/en/reference/testing") + } else if stat.IsDir() { + tests, err := os.ReadDir(rootPath) + if err != nil { + return 0, nil, errHint(err, "See https://docs.vespa.ai/en/reference/testing") + } + context := testContext{testsPath: rootPath, dryRun: dryRun, cli: cli} + previousFailed := false + for _, test := range tests { + if !test.IsDir() && filepath.Ext(test.Name()) == ".json" { + testPath := filepath.Join(rootPath, test.Name()) + if previousFailed { + fmt.Fprintln(cli.Stdout, "") + previousFailed = false + } + failure, err := runTest(testPath, context) + if err != nil { + return 0, nil, err + } + if failure != "" { + failed = append(failed, failure) + previousFailed = true + } + count++ + } + } + } else if strings.HasSuffix(stat.Name(), ".json") { + failure, err := runTest(rootPath, testContext{testsPath: filepath.Dir(rootPath), dryRun: dryRun, cli: cli}) + if err != nil { + return 0, nil, err + } + if failure != "" { + failed = append(failed, failure) + } + count++ + } + if count == 0 { + return 0, nil, errHint(fmt.Errorf("failed to find any tests at %s", rootPath), "See https://docs.vespa.ai/en/reference/testing") + } + return count, failed, nil +} + +// Runs the test at the given path, and returns the specified test name if the test fails +func runTest(testPath string, context testContext) (string, error) { + var test test + testBytes, err := os.ReadFile(testPath) + if err != nil { + return "", errHint(err, "See https://docs.vespa.ai/en/reference/testing") + } + if err = json.Unmarshal(testBytes, &test); err != nil { + return "", errHint(fmt.Errorf("failed parsing test at %s: %w", testPath, err), "See https://docs.vespa.ai/en/reference/testing") + } + + testName := test.Name + if test.Name == "" { + testName = filepath.Base(testPath) + } + if !context.dryRun { + fmt.Fprintf(context.cli.Stdout, "%s:", testName) + } + + defaultParameters, err := getParameters(test.Defaults.ParametersRaw, filepath.Dir(testPath)) + if err != nil { + fmt.Fprintln(context.cli.Stderr) + return "", errHint(fmt.Errorf("invalid default parameters for %s: %w", testName, err), "See https://docs.vespa.ai/en/reference/testing") + } + + if len(test.Steps) == 0 { + fmt.Fprintln(context.cli.Stderr) + return "", errHint(fmt.Errorf("a test must have at least one step, but none were found in %s", testPath), "See https://docs.vespa.ai/en/reference/testing") + } + for i, step := range test.Steps { + stepName := fmt.Sprintf("Step %d", i+1) + if step.Name != "" { + stepName += ": " + step.Name + } + failure, longFailure, err := verify(step, test.Defaults.Cluster, defaultParameters, context) + if err != nil { + fmt.Fprintln(context.cli.Stderr) + return "", errHint(fmt.Errorf("error in %s: %w", stepName, err), "See https://docs.vespa.ai/en/reference/testing") + } + if !context.dryRun { + if failure != "" { + fmt.Fprintf(context.cli.Stdout, " %s\n%s:\n%s\n", color.RedString("failed"), stepName, longFailure) + return fmt.Sprintf("%s: %s: %s", testName, stepName, failure), nil + } + if i == 0 { + fmt.Fprintf(context.cli.Stdout, " ") + } + fmt.Fprint(context.cli.Stdout, ".") + } + } + if !context.dryRun { + fmt.Fprintln(context.cli.Stdout, color.GreenString(" OK")) + } + return "", nil +} + +// Asserts specified response is obtained for request, or returns a failure message, or an error if this fails +func verify(step step, defaultCluster string, defaultParameters map[string]string, context testContext) (string, string, error) { + requestBody, err := getBody(step.Request.BodyRaw, context.testsPath) + if err != nil { + return "", "", err + } + + parameters, err := getParameters(step.Request.ParametersRaw, context.testsPath) + if err != nil { + return "", "", err + } + for name, value := range defaultParameters { + if _, present := parameters[name]; !present { + parameters[name] = value + } + } + + cluster := step.Request.Cluster + if cluster == "" { + cluster = defaultCluster + } + + method := step.Request.Method + if method == "" { + method = "GET" + } + + var service *vespa.Service + requestUri := step.Request.URI + if requestUri == "" { + requestUri = "/search/" + } + requestUrl, err := url.ParseRequestURI(requestUri) + if err != nil { + return "", "", err + } + externalEndpoint := requestUrl.IsAbs() + if !externalEndpoint && filepath.Base(context.testsPath) == "production-test" { + return "", "", fmt.Errorf("production tests may not specify requests against Vespa endpoints") + } + if !externalEndpoint && !context.dryRun { + target, err := context.target() + if err != nil { + return "", "", err + } + service, err = target.Service(vespa.QueryService, 0, 0, cluster) + if err != nil { + return "", "", err + } + requestUrl, err = url.ParseRequestURI(service.BaseURL + requestUri) + if err != nil { + return "", "", err + } + } + query := requestUrl.Query() + for name, value := range parameters { + query.Add(name, value) + } + requestUrl.RawQuery = query.Encode() + + header := http.Header{} + header.Add("Content-Type", "application/json") // TODO: Not guaranteed to be true ... + + request := &http.Request{ + URL: requestUrl, + Method: method, + Header: header, + Body: io.NopCloser(bytes.NewReader(requestBody)), + } + defer request.Body.Close() + + statusCode := step.Response.Code + if statusCode == 0 { + statusCode = 200 + } + + responseBodySpecBytes, err := getBody(step.Response.BodyRaw, context.testsPath) + if err != nil { + return "", "", err + } + var responseBodySpec interface{} + if responseBodySpecBytes != nil { + err = json.Unmarshal(responseBodySpecBytes, &responseBodySpec) + if err != nil { + return "", "", fmt.Errorf("invalid response body spec: %w", err) + } + } + + if context.dryRun { + return "", "", nil + } + + var response *http.Response + if externalEndpoint { + context.cli.httpClient.UseCertificate([]tls.Certificate{}) + response, err = context.cli.httpClient.Do(request, 60*time.Second) + } else { + response, err = service.Do(request, 600*time.Second) // Vespa should provide a response within the given request timeout + } + if err != nil { + return "", "", err + } + defer response.Body.Close() + + if statusCode != response.StatusCode { + return fmt.Sprintf("Unexpected status code: %s", color.RedString(strconv.Itoa(response.StatusCode))), + fmt.Sprintf("Unexpected status code\nExpected: %s\nActual: %s\nRequested: %s at %s\nResponse:\n%s", + color.CyanString(strconv.Itoa(statusCode)), + color.RedString(strconv.Itoa(response.StatusCode)), + color.CyanString(method), + color.CyanString(requestUrl.String()), + util.ReaderToJSON(response.Body)), nil + } + + if responseBodySpec == nil { + return "", "", nil + } + + responseBodyBytes, err := io.ReadAll(response.Body) + if err != nil { + return "", "", err + } + var responseBody interface{} + err = json.Unmarshal(responseBodyBytes, &responseBody) + if err != nil { + return "", "", fmt.Errorf("got non-JSON response; %w:\n%s", err, string(responseBodyBytes)) + } + + failure, expected, actual, err := compare(responseBodySpec, responseBody, "") + if failure != "" { + responsePretty, _ := json.MarshalIndent(responseBody, "", " ") + longFailure := failure + if expected != "" { + longFailure += "\nExpected: " + expected + } + if actual != "" { + failure += ": " + actual + longFailure += "\nActual: " + actual + } + longFailure += fmt.Sprintf("\nRequested: %s at %s\nResponse:\n%s", color.CyanString(method), color.CyanString(requestUrl.String()), string(responsePretty)) + return failure, longFailure, err + } + return "", "", err +} + +func compare(expected interface{}, actual interface{}, path string) (string, string, string, error) { + typeMatch := false + valueMatch := false + switch u := expected.(type) { + case nil: + typeMatch = actual == nil + valueMatch = actual == nil + case bool: + v, ok := actual.(bool) + typeMatch = ok + valueMatch = ok && u == v + case float64: + v, ok := actual.(float64) + typeMatch = ok + valueMatch = ok && math.Abs(u-v) < 1e-9 + case string: + v, ok := actual.(string) + typeMatch = ok + valueMatch = ok && (u == v) + case []interface{}: + v, ok := actual.([]interface{}) + typeMatch = ok + if ok { + if len(u) == len(v) { + for i, e := range u { + if failure, expected, actual, err := compare(e, v[i], fmt.Sprintf("%s/%d", path, i)); failure != "" || err != nil { + return failure, expected, actual, err + } + } + valueMatch = true + } else { + return fmt.Sprintf("Unexpected number of elements at %s", color.CyanString(path)), + color.CyanString(strconv.Itoa(len(u))), + color.RedString(strconv.Itoa(len(v))), + nil + } + } + case map[string]interface{}: + v, ok := actual.(map[string]interface{}) + typeMatch = ok + if ok { + for n, e := range u { + childPath := fmt.Sprintf("%s/%s", path, strings.ReplaceAll(strings.ReplaceAll(n, "~", "~0"), "/", "~1")) + f, ok := v[n] + if !ok { + return fmt.Sprintf("Missing expected field at %s", color.RedString(childPath)), "", "", nil + } + if failure, expected, actual, err := compare(e, f, childPath); failure != "" || err != nil { + return failure, expected, actual, err + } + } + valueMatch = true + } + default: + return "", "", "", fmt.Errorf("unexpected JSON type for value '%v'", expected) + } + + if !valueMatch { + if path == "" { + path = "root" + } + mismatched := "type" + if typeMatch { + mismatched = "value" + } + expectedJson, _ := json.Marshal(expected) + actualJson, _ := json.Marshal(actual) + return fmt.Sprintf("Unexpected %s at %s", mismatched, color.CyanString(path)), + color.CyanString(string(expectedJson)), + color.RedString(string(actualJson)), + nil + } + return "", "", "", nil +} + +func getParameters(parametersRaw []byte, testsPath string) (map[string]string, error) { + if parametersRaw != nil { + var parametersPath string + if err := json.Unmarshal(parametersRaw, ¶metersPath); err == nil { + if err = validateRelativePath(parametersPath); err != nil { + return nil, err + } + resolvedParametersPath := filepath.Join(testsPath, parametersPath) + parametersRaw, err = os.ReadFile(resolvedParametersPath) + if err != nil { + return nil, fmt.Errorf("failed to read request parameters at %s: %w", resolvedParametersPath, err) + } + } + var parameters map[string]string + if err := json.Unmarshal(parametersRaw, ¶meters); err != nil { + return nil, fmt.Errorf("request parameters must be JSON with only string values: %w", err) + } + return parameters, nil + } + return make(map[string]string), nil +} + +func getBody(bodyRaw []byte, testsPath string) ([]byte, error) { + var bodyPath string + if err := json.Unmarshal(bodyRaw, &bodyPath); err == nil { + if err = validateRelativePath(bodyPath); err != nil { + return nil, err + } + resolvedBodyPath := filepath.Join(testsPath, bodyPath) + bodyRaw, err = os.ReadFile(resolvedBodyPath) + if err != nil { + return nil, fmt.Errorf("failed to read body file at %s: %w", resolvedBodyPath, err) + } + } + return bodyRaw, nil +} + +func validateRelativePath(relPath string) error { + if filepath.IsAbs(relPath) { + return fmt.Errorf("path must be relative, but was '%s'", relPath) + } + cleanPath := filepath.Clean(relPath) + if strings.HasPrefix(cleanPath, "../../../") { + return fmt.Errorf("path may not point outside src/test/application, but '%s' does", relPath) + } + return nil +} + +type test struct { + Name string `json:"name"` + Defaults defaults `json:"defaults"` + Steps []step `json:"steps"` +} + +type defaults struct { + Cluster string `json:"cluster"` + ParametersRaw json.RawMessage `json:"parameters"` +} + +type step struct { + Name string `json:"name"` + Request request `json:"request"` + Response response `json:"response"` +} + +type request struct { + Cluster string `json:"cluster"` + Method string `json:"method"` + URI string `json:"uri"` + ParametersRaw json.RawMessage `json:"parameters"` + BodyRaw json.RawMessage `json:"body"` +} + +type response struct { + Code int `json:"code"` + BodyRaw json.RawMessage `json:"body"` +} + +type testContext struct { + cli *CLI + lazyTarget vespa.Target + testsPath string + dryRun bool +} + +func (t *testContext) target() (vespa.Target, error) { + if t.lazyTarget == nil { + target, err := t.cli.target(targetOptions{}) + if err != nil { + return nil, err + } + t.lazyTarget = target + } + return t.lazyTarget, nil +} diff --git a/client/go/internal/cli/cmd/test_test.go b/client/go/internal/cli/cmd/test_test.go new file mode 100644 index 00000000000..5d6bb441b2a --- /dev/null +++ b/client/go/internal/cli/cmd/test_test.go @@ -0,0 +1,183 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// test command tests +// Author: jonmv + +package cmd + +import ( + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vespa-engine/vespa/client/go/internal/mock" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func TestSuite(t *testing.T) { + client := &mock.HTTPClient{} + searchResponse, _ := os.ReadFile("testdata/tests/response.json") + client.NextStatus(200) + client.NextStatus(200) + for i := 0; i < 11; i++ { + client.NextResponseString(200, string(searchResponse)) + } + + expectedBytes, _ := os.ReadFile("testdata/tests/expected-suite.out") + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = client + assert.NotNil(t, cli.Run("test", "testdata/tests/system-test")) + + baseUrl := "http://127.0.0.1:8080" + urlWithQuery := baseUrl + "/search/?presentation.timing=true&query=artist%3A+foo&timeout=3.4s" + requests := []*http.Request{createFeedRequest(baseUrl), createFeedRequest(baseUrl), createSearchRequest(urlWithQuery), createSearchRequest(urlWithQuery)} + requests = append(requests, createSearchRequest(baseUrl+"/search/")) + requests = append(requests, createSearchRequest(baseUrl+"/search/?foo=%2F")) + for i := 0; i < 7; i++ { + requests = append(requests, createSearchRequest(baseUrl+"/search/")) + } + assertRequests(requests, client, t) + assert.Equal(t, string(expectedBytes), stdout.String()) + assert.Equal(t, "", stderr.String()) +} + +func TestIllegalFileReference(t *testing.T) { + client := &mock.HTTPClient{} + client.NextStatus(200) + client.NextStatus(200) + cli, _, stderr := newTestCLI(t) + cli.httpClient = client + assert.NotNil(t, cli.Run("test", "testdata/tests/production-test/illegal-reference.json")) + assertRequests([]*http.Request{createRequest("GET", "https://domain.tld", "{}")}, client, t) + assert.Equal(t, "\nError: error in Step 2: path may not point outside src/test/application, but 'foo/../../../../this-is-not-ok.json' does\nHint: See https://docs.vespa.ai/en/reference/testing\n", stderr.String()) +} + +func TestIllegalRequestUri(t *testing.T) { + client := &mock.HTTPClient{} + client.NextStatus(200) + client.NextStatus(200) + cli, _, stderr := newTestCLI(t) + cli.httpClient = client + assert.NotNil(t, cli.Run("test", "testdata/tests/production-test/illegal-uri.json")) + assertRequests([]*http.Request{createRequest("GET", "https://domain.tld/my-api", "")}, client, t) + assert.Equal(t, "\nError: error in Step 2: production tests may not specify requests against Vespa endpoints\nHint: See https://docs.vespa.ai/en/reference/testing\n", stderr.String()) +} + +func TestProductionTest(t *testing.T) { + client := &mock.HTTPClient{} + client.NextStatus(200) + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = client + assert.Nil(t, cli.Run("test", "testdata/tests/production-test/external.json")) + assert.Equal(t, "external.json: . OK\n\nSuccess: 1 test OK\n", stdout.String()) + assert.Equal(t, "", stderr.String()) + assertRequests([]*http.Request{createRequest("GET", "https://my.service:123/path?query=wohoo", "")}, client, t) +} + +func TestTestWithoutAssertions(t *testing.T) { + cli, _, stderr := newTestCLI(t) + assert.NotNil(t, cli.Run("test", "testdata/tests/system-test/foo/query.json")) + assert.Equal(t, "\nError: a test must have at least one step, but none were found in testdata/tests/system-test/foo/query.json\nHint: See https://docs.vespa.ai/en/reference/testing\n", stderr.String()) +} + +func TestSuiteWithoutTests(t *testing.T) { + cli, _, stderr := newTestCLI(t) + assert.NotNil(t, cli.Run("test", "testdata/tests/staging-test")) + assert.Equal(t, "Error: failed to find any tests at testdata/tests/staging-test\nHint: See https://docs.vespa.ai/en/reference/testing\n", stderr.String()) +} + +func TestSingleTest(t *testing.T) { + client := &mock.HTTPClient{} + searchResponse, _ := os.ReadFile("testdata/tests/response.json") + client.NextStatus(200) + client.NextStatus(200) + client.NextResponseString(200, string(searchResponse)) + client.NextResponseString(200, string(searchResponse)) + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = client + + expectedBytes, _ := os.ReadFile("testdata/tests/expected.out") + assert.Nil(t, cli.Run("test", "testdata/tests/system-test/test.json")) + assert.Equal(t, string(expectedBytes), stdout.String()) + assert.Equal(t, "", stderr.String()) + + baseUrl := "http://127.0.0.1:8080" + rawUrl := baseUrl + "/search/?presentation.timing=true&query=artist%3A+foo&timeout=3.4s" + assertRequests([]*http.Request{createFeedRequest(baseUrl), createFeedRequest(baseUrl), createSearchRequest(rawUrl), createSearchRequest(rawUrl)}, client, t) +} + +func TestSingleTestWithCloudAndEndpoints(t *testing.T) { + apiKey, err := vespa.CreateAPIKey() + require.Nil(t, err) + certDir := filepath.Join(t.TempDir()) + keyFile := filepath.Join(certDir, "key") + certFile := filepath.Join(certDir, "cert") + kp, err := vespa.CreateKeyPair() + require.Nil(t, err) + require.Nil(t, os.WriteFile(keyFile, kp.PrivateKey, 0600)) + require.Nil(t, os.WriteFile(certFile, kp.Certificate, 0600)) + + client := &mock.HTTPClient{} + cli, stdout, stderr := newTestCLI( + t, + "VESPA_CLI_API_KEY="+string(apiKey), + "VESPA_CLI_DATA_PLANE_KEY_FILE="+keyFile, + "VESPA_CLI_DATA_PLANE_CERT_FILE="+certFile, + "VESPA_CLI_ENDPOINTS={\"endpoints\":[{\"cluster\":\"container\",\"url\":\"https://url\"}]}", + ) + cli.httpClient = client + + searchResponse, err := os.ReadFile("testdata/tests/response.json") + require.Nil(t, err) + client.NextStatus(200) + client.NextStatus(200) + client.NextResponseString(200, string(searchResponse)) + client.NextResponseString(200, string(searchResponse)) + + assert.Nil(t, cli.Run("test", "testdata/tests/system-test/test.json", "-t", "cloud", "-a", "t.a.i")) + expectedBytes, err := os.ReadFile("testdata/tests/expected.out") + require.Nil(t, err) + assert.Equal(t, "", stderr.String()) + assert.Equal(t, string(expectedBytes), stdout.String()) + + baseUrl := "https://url" + rawUrl := baseUrl + "/search/?presentation.timing=true&query=artist%3A+foo&timeout=3.4s" + assertRequests([]*http.Request{createFeedRequest(baseUrl), createFeedRequest(baseUrl), createSearchRequest(rawUrl), createSearchRequest(rawUrl)}, client, t) +} + +func createFeedRequest(urlPrefix string) *http.Request { + return createRequest("POST", + urlPrefix+"/document/v1/test/music/docid/doc?timeout=3.4s", + "{\"fields\":{\"artist\":\"Foo Fighters\"}}") +} + +func createSearchRequest(rawUrl string) *http.Request { + return createRequest("GET", rawUrl, "") +} + +func createRequest(method string, uri string, body string) *http.Request { + requestUrl, _ := url.ParseRequestURI(uri) + return &http.Request{ + URL: requestUrl, + Method: method, + Header: nil, + Body: io.NopCloser(strings.NewReader(body)), + } +} + +func assertRequests(requests []*http.Request, client *mock.HTTPClient, t *testing.T) { + if assert.Equal(t, len(requests), len(client.Requests)) { + for i, e := range requests { + a := client.Requests[i] + assert.Equal(t, e.URL.String(), a.URL.String()) + assert.Equal(t, e.Method, a.Method) + assert.Equal(t, util.ReaderToJSON(e.Body), util.ReaderToJSON(a.Body)) + } + } +} diff --git a/client/go/internal/cli/cmd/testdata/A-Head-Full-of-Dreams-Put.json b/client/go/internal/cli/cmd/testdata/A-Head-Full-of-Dreams-Put.json new file mode 100644 index 00000000000..c67b90664d6 --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/A-Head-Full-of-Dreams-Put.json @@ -0,0 +1,15 @@ +{ + "put": "id:mynamespace:music::a-head-full-of-dreams", + "fields": { + "album": "A Head Full of Dreams", + "artist": "Coldplay", + "year": 2015, + "category_scores": { + "cells": [ + { "address" : { "cat" : "pop" }, "value": 1 }, + { "address" : { "cat" : "rock" }, "value": 0.2 }, + { "address" : { "cat" : "jazz" }, "value": 0 } + ] + } + } +} diff --git a/client/go/internal/cli/cmd/testdata/A-Head-Full-of-Dreams-Remove.json b/client/go/internal/cli/cmd/testdata/A-Head-Full-of-Dreams-Remove.json new file mode 100644 index 00000000000..8eeb19cbaa6 --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/A-Head-Full-of-Dreams-Remove.json @@ -0,0 +1,3 @@ +{ + "remove": "id:mynamespace:music::a-head-full-of-dreams" +} diff --git a/client/go/internal/cli/cmd/testdata/A-Head-Full-of-Dreams-Update.json b/client/go/internal/cli/cmd/testdata/A-Head-Full-of-Dreams-Update.json new file mode 100644 index 00000000000..0f5e35c3fd5 --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/A-Head-Full-of-Dreams-Update.json @@ -0,0 +1,12 @@ +{ + "update": "id:mynamespace:music::a-head-full-of-dreams", + "fields": { + "category_scores": { + "cells": [ + { "address" : { "cat" : "pop" }, "value": 1 }, + { "address" : { "cat" : "rock" }, "value": 0.2 }, + { "address" : { "cat" : "jazz" }, "value": 0 } + ] + } + } +} diff --git a/client/go/internal/cli/cmd/testdata/A-Head-Full-of-Dreams-Without-Operation.json b/client/go/internal/cli/cmd/testdata/A-Head-Full-of-Dreams-Without-Operation.json new file mode 100644 index 00000000000..b68872a961e --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/A-Head-Full-of-Dreams-Without-Operation.json @@ -0,0 +1,14 @@ +{ + "fields": { + "album": "A Head Full of Dreams", + "artist": "Coldplay", + "year": 2015, + "category_scores": { + "cells": [ + { "address" : { "cat" : "pop" }, "value": 1 }, + { "address" : { "cat" : "rock" }, "value": 0.2 }, + { "address" : { "cat" : "jazz" }, "value": 0 } + ] + } + } +} diff --git a/client/go/internal/cli/cmd/testdata/applications/withDeployment/target/application-test.zip b/client/go/internal/cli/cmd/testdata/applications/withDeployment/target/application-test.zip Binary files differnew file mode 100644 index 00000000000..8a1707b9cee --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/applications/withDeployment/target/application-test.zip diff --git a/client/go/internal/cli/cmd/testdata/applications/withDeployment/target/application.zip b/client/go/internal/cli/cmd/testdata/applications/withDeployment/target/application.zip Binary files differnew file mode 100644 index 00000000000..da23c2ff437 --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/applications/withDeployment/target/application.zip diff --git a/client/go/internal/cli/cmd/testdata/applications/withEmptyTarget/pom.xml b/client/go/internal/cli/cmd/testdata/applications/withEmptyTarget/pom.xml new file mode 100644 index 00000000000..bb7a232ea2a --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/applications/withEmptyTarget/pom.xml @@ -0,0 +1,2 @@ +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services/> diff --git a/client/go/internal/cli/cmd/testdata/applications/withEmptyTarget/target/placeholder b/client/go/internal/cli/cmd/testdata/applications/withEmptyTarget/target/placeholder new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/applications/withEmptyTarget/target/placeholder diff --git a/client/go/internal/cli/cmd/testdata/applications/withInvalidEntries/target/application-test.zip b/client/go/internal/cli/cmd/testdata/applications/withInvalidEntries/target/application-test.zip Binary files differnew file mode 100644 index 00000000000..3e4e8c23f9b --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/applications/withInvalidEntries/target/application-test.zip diff --git a/client/go/internal/cli/cmd/testdata/applications/withInvalidEntries/target/application.zip b/client/go/internal/cli/cmd/testdata/applications/withInvalidEntries/target/application.zip Binary files differnew file mode 100644 index 00000000000..3e4e8c23f9b --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/applications/withInvalidEntries/target/application.zip diff --git a/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/hosts.xml b/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/hosts.xml new file mode 100644 index 00000000000..4b4f607ea95 --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/hosts.xml @@ -0,0 +1,8 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<hosts> + <host name="localhost"> + <alias>node1</alias> + </host> +</hosts> + diff --git a/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/schemas/msmarco.sd b/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/schemas/msmarco.sd new file mode 100644 index 00000000000..3c481edc10d --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/schemas/msmarco.sd @@ -0,0 +1,299 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +schema msmarco { + document msmarco { + + field id type string { + indexing: summary | attribute + } + + field title type string { + indexing: index | summary + index: enable-bm25 + stemming: best + } + + field url type string { + indexing: index | summary + } + + field body type string { + indexing: index | summary + index: enable-bm25 + summary: dynamic + stemming: best + } + + field title_word2vec type tensor<float>(x[500]) { + indexing: attribute + } + + field body_word2vec type tensor<float>(x[500]) { + indexing: attribute + } + + field title_gse type tensor<float>(x[512]) { + indexing: attribute + } + + field body_gse type tensor<float>(x[512]) { + indexing: attribute + } + + field title_bert type tensor<float>(x[768]) { + indexing: attribute + } + + field body_bert type tensor<float>(x[768]) { + indexing: attribute + } + + } + + document-summary minimal { + summary id type string {} + } + + fieldset default { + fields: title, body + } + + rank-profile default { + first-phase { + expression: nativeRank(title, body) + } + } + + rank-profile bm25 inherits default { + first-phase { + expression: bm25(title) + bm25(body) + } + } + + rank-profile word2vec_title_body_all inherits default { + function dot_product_title() { + expression: sum(query(tensor)*attribute(title_word2vec)) + } + function dot_product_body() { + expression: sum(query(tensor)*attribute(body_word2vec)) + } + first-phase { + expression: dot_product_title() + dot_product_body() + } + ignore-default-rank-features + rank-features { + rankingExpression(dot_product_title) + rankingExpression(dot_product_body) + } + } + + rank-profile gse_title_body_all inherits default { + function dot_product_title() { + expression: sum(query(tensor_gse)*attribute(title_gse)) + } + function dot_product_body() { + expression: sum(query(tensor_gse)*attribute(body_gse)) + } + first-phase { + expression: dot_product_title() + dot_product_body() + } + ignore-default-rank-features + rank-features { + rankingExpression(dot_product_title) + rankingExpression(dot_product_body) + } + } + + rank-profile bert_title_body_all inherits default { + function dot_product_title() { + expression: sum(query(tensor_bert)*attribute(title_bert)) + } + function dot_product_body() { + expression: sum(query(tensor_bert)*attribute(body_bert)) + } + first-phase { + expression: dot_product_title() + dot_product_body() + } + ignore-default-rank-features + rank-features { + rankingExpression(dot_product_title) + rankingExpression(dot_product_body) + } + } + + rank-profile bm25_word2vec_title_body_all inherits default { + function dot_product_title() { + expression: sum(query(tensor)*attribute(title_word2vec)) + } + function dot_product_body() { + expression: sum(query(tensor)*attribute(body_word2vec)) + } + first-phase { + expression: bm25(title) + bm25(body) + dot_product_title() + dot_product_body() + } + ignore-default-rank-features + rank-features { + bm25(title) + bm25(body) + rankingExpression(dot_product_title) + rankingExpression(dot_product_body) + } + } + + rank-profile bm25_gse_title_body_all inherits default { + function dot_product_title() { + expression: sum(query(tensor_gse)*attribute(title_gse)) + } + function dot_product_body() { + expression: sum(query(tensor_gse)*attribute(body_gse)) + } + first-phase { + expression: bm25(title) + bm25(body) + dot_product_title() + dot_product_body() + } + ignore-default-rank-features + rank-features { + bm25(title) + bm25(body) + rankingExpression(dot_product_title) + rankingExpression(dot_product_body) + } + } + + rank-profile bm25_bert_title_body_all inherits default { + function dot_product_title() { + expression: sum(query(tensor_bert)*attribute(title_bert)) + } + function dot_product_body() { + expression: sum(query(tensor_bert)*attribute(body_bert)) + } + first-phase { + expression: bm25(title) + bm25(body) + dot_product_title() + dot_product_body() + } + ignore-default-rank-features + rank-features { + bm25(title) + bm25(body) + rankingExpression(dot_product_title) + rankingExpression(dot_product_body) + } + } + + rank-profile listwise_bm25_bert_title_body_all inherits default { + function dot_product_title() { + expression: sum(query(tensor_bert)*attribute(title_bert)) + } + function dot_product_body() { + expression: sum(query(tensor_bert)*attribute(body_bert)) + } + first-phase { + expression: 0.9005951 * bm25(title) + 2.2043643 * bm25(body) + 0.13506432 * dot_product_title() + 0.5840874 * dot_product_body() + } + ignore-default-rank-features + rank-features { + bm25(title) + bm25(body) + rankingExpression(dot_product_title) + rankingExpression(dot_product_body) + } + } + + rank-profile listwise_linear_bm25_gse_title_body_and inherits default { + function dot_product_title() { + expression: sum(query(tensor_gse)*attribute(title_gse)) + } + function dot_product_body() { + expression: sum(query(tensor_gse)*attribute(body_gse)) + } + first-phase { + expression: 0.12408562 * bm25(title) + 0.36673144 * bm25(body) + 6.2273498 * dot_product_title() + 5.671119 * dot_product_body() + } + ignore-default-rank-features + rank-features { + bm25(title) + bm25(body) + rankingExpression(dot_product_title) + rankingExpression(dot_product_body) + } + } + + rank-profile listwise_linear_bm25_gse_title_body_or inherits default { + function dot_product_title() { + expression: sum(query(tensor_gse)*attribute(title_gse)) + } + function dot_product_body() { + expression: sum(query(tensor_gse)*attribute(body_gse)) + } + first-phase { + expression: 0.7150663 * bm25(title) + 0.9480147 * bm25(body) + 1.560068 * dot_product_title() + 1.5062317 * dot_product_body() + } + ignore-default-rank-features + rank-features { + bm25(title) + bm25(body) + rankingExpression(dot_product_title) + rankingExpression(dot_product_body) + } + } + + rank-profile pointwise_linear_bm25 inherits default { + first-phase { + expression: 0.22499913 * bm25(title) + 0.07596389 * bm25(body) + } + } + + rank-profile listwise_linear_bm25 inherits default { + first-phase { + expression: 0.13446581 * bm25(title) + 0.5716889 * bm25(body) + } + } + + rank-profile collect_rank_features_embeddings inherits default { + function dot_product_title_word2vec() { + expression: sum(query(tensor)*attribute(title_word2vec)) + } + function dot_product_body_word2vec() { + expression: sum(query(tensor)*attribute(body_word2vec)) + } + function dot_product_title_gse() { + expression: sum(query(tensor_gse)*attribute(title_gse)) + } + function dot_product_body_gse() { + expression: sum(query(tensor_gse)*attribute(body_gse)) + } + function dot_product_title_bert() { + expression: sum(query(tensor_bert)*attribute(title_bert)) + } + function dot_product_body_bert() { + expression: sum(query(tensor_bert)*attribute(body_bert)) + } + first-phase { + expression: random + } + ignore-default-rank-features + rank-features { + bm25(title) + bm25(body) + nativeRank(title) + nativeRank(body) + rankingExpression(dot_product_title_word2vec) + rankingExpression(dot_product_body_word2vec) + rankingExpression(dot_product_title_gse) + rankingExpression(dot_product_body_gse) + rankingExpression(dot_product_title_bert) + rankingExpression(dot_product_body_bert) + } + } + + rank-profile collect_rank_features inherits default { + first-phase { + expression: random + } + ignore-default-rank-features + rank-features { + bm25(title) + bm25(body) + nativeRank(title) + nativeRank(body) + } + } +} diff --git a/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/services.xml b/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/services.xml new file mode 100644 index 00000000000..60267517c33 --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/services.xml @@ -0,0 +1,62 @@ +<?xml version='1.0' encoding='UTF-8'?> +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + +<services version="1.0"> + + <container id="text_search" version="1.0"> + <document-api/> + <search> + + <!-- Config for bolding in search result snippets --> + <config name="container.qr-searchers"> + <tag> + <bold> + <open><strong></open> + <close></strong></close> + </bold> + <separator>...</separator> + </tag> + </config> + + </search> + <document-processing/> + + <component id="com.yahoo.language.simple.SimpleLinguistics"/> + + <handler id="ai.vespa.example.text_search.site.SiteHandler" bundle="text-search"> + <binding>http://*/site/*</binding> + <binding>http://*/site</binding> + <config name="ai.vespa.example.text_search.site.site-handler"> + <vespaHostName>localhost</vespaHostName> + <vespaHostPort>8080</vespaHostPort> + </config> + </handler> + + <nodes> + <node hostalias="node1" /> + <jvm options="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=*:8998"/> + </nodes> + + </container> + + <content id="msmarco" version="1.0"> + + <!-- Config for search result snippets --> + <config name="vespa.config.search.summary.juniperrc"> + <max_matches>2</max_matches> + <length>1000</length> + <surround_max>500</surround_max> + <min_length>300</min_length> + </config> + + <redundancy>2</redundancy> + <documents> + <document type='msmarco' mode="index"/> + <document-processing cluster="text_search"/> + </documents> + <nodes> + <node distribution-key='0' hostalias='node1'/> + </nodes> + </content> + +</services> diff --git a/client/go/internal/cli/cmd/testdata/applications/withTarget/pom.xml b/client/go/internal/cli/cmd/testdata/applications/withTarget/pom.xml new file mode 100644 index 00000000000..bb7a232ea2a --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/applications/withTarget/pom.xml @@ -0,0 +1,2 @@ +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services/> diff --git a/client/go/internal/cli/cmd/testdata/applications/withTarget/target/application.zip b/client/go/internal/cli/cmd/testdata/applications/withTarget/target/application.zip Binary files differnew file mode 100644 index 00000000000..b017db6472d --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/applications/withTarget/target/application.zip diff --git a/client/go/internal/cli/cmd/testdata/empty.json b/client/go/internal/cli/cmd/testdata/empty.json new file mode 100644 index 00000000000..9e26dfeeb6e --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/empty.json @@ -0,0 +1 @@ +{}
\ No newline at end of file diff --git a/client/go/internal/cli/cmd/testdata/sample-apps-contents.json b/client/go/internal/cli/cmd/testdata/sample-apps-contents.json new file mode 100644 index 00000000000..02824f01147 --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/sample-apps-contents.json @@ -0,0 +1,610 @@ +[ + { + "name": ".github", + "path": ".github", + "sha": "6bcd814282757c3ca7ba265972f000bc62a76aa8", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/.github?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/.github", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/6bcd814282757c3ca7ba265972f000bc62a76aa8", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/.github?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/6bcd814282757c3ca7ba265972f000bc62a76aa8", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/.github" + } + }, + { + "name": ".gitignore", + "path": ".gitignore", + "sha": "e33d8f27d17a696ae99b12063f553a53d0d08510", + "size": 163, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/.gitignore?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/blob/master/.gitignore", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/e33d8f27d17a696ae99b12063f553a53d0d08510", + "download_url": "https://raw.githubusercontent.com/vespa-engine/sample-apps/master/.gitignore", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/.gitignore?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/e33d8f27d17a696ae99b12063f553a53d0d08510", + "html": "https://github.com/vespa-engine/sample-apps/blob/master/.gitignore" + } + }, + { + "name": "CONTRIBUTING.md", + "path": "CONTRIBUTING.md", + "sha": "56d974d3c4d9b5b8f1739c24fc293f822e2fcec8", + "size": 3814, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/CONTRIBUTING.md?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/blob/master/CONTRIBUTING.md", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/56d974d3c4d9b5b8f1739c24fc293f822e2fcec8", + "download_url": "https://raw.githubusercontent.com/vespa-engine/sample-apps/master/CONTRIBUTING.md", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/CONTRIBUTING.md?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/56d974d3c4d9b5b8f1739c24fc293f822e2fcec8", + "html": "https://github.com/vespa-engine/sample-apps/blob/master/CONTRIBUTING.md" + } + }, + { + "name": "Code-of-Conduct.md", + "path": "Code-of-Conduct.md", + "sha": "a812e1cfa8e464fc4a0ba6743030662efa1e4244", + "size": 7414, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/Code-of-Conduct.md?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/blob/master/Code-of-Conduct.md", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/a812e1cfa8e464fc4a0ba6743030662efa1e4244", + "download_url": "https://raw.githubusercontent.com/vespa-engine/sample-apps/master/Code-of-Conduct.md", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/Code-of-Conduct.md?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/a812e1cfa8e464fc4a0ba6743030662efa1e4244", + "html": "https://github.com/vespa-engine/sample-apps/blob/master/Code-of-Conduct.md" + } + }, + { + "name": "Gemfile", + "path": "Gemfile", + "sha": "a7befcbb3c5a4ff65d51ee222c27bc3aef92bdc8", + "size": 308, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/Gemfile?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/blob/master/Gemfile", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/a7befcbb3c5a4ff65d51ee222c27bc3aef92bdc8", + "download_url": "https://raw.githubusercontent.com/vespa-engine/sample-apps/master/Gemfile", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/Gemfile?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/a7befcbb3c5a4ff65d51ee222c27bc3aef92bdc8", + "html": "https://github.com/vespa-engine/sample-apps/blob/master/Gemfile" + } + }, + { + "name": "LICENSE", + "path": "LICENSE", + "sha": "df0bb94aeb6578299d1995c791da29f8f00f0cbc", + "size": 11354, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/LICENSE?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/blob/master/LICENSE", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/df0bb94aeb6578299d1995c791da29f8f00f0cbc", + "download_url": "https://raw.githubusercontent.com/vespa-engine/sample-apps/master/LICENSE", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/LICENSE?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/df0bb94aeb6578299d1995c791da29f8f00f0cbc", + "html": "https://github.com/vespa-engine/sample-apps/blob/master/LICENSE" + } + }, + { + "name": "README.md", + "path": "README.md", + "sha": "44b3d846b1c594196d1362a44c965b91482fb806", + "size": 8401, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/README.md?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/blob/master/README.md", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/44b3d846b1c594196d1362a44c965b91482fb806", + "download_url": "https://raw.githubusercontent.com/vespa-engine/sample-apps/master/README.md", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/README.md?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/44b3d846b1c594196d1362a44c965b91482fb806", + "html": "https://github.com/vespa-engine/sample-apps/blob/master/README.md" + } + }, + { + "name": "Vagrantfile", + "path": "Vagrantfile", + "sha": "0cb4b6dbe12404fb0bfa67e8e950287cdad722f4", + "size": 913, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/Vagrantfile?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/blob/master/Vagrantfile", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/0cb4b6dbe12404fb0bfa67e8e950287cdad722f4", + "download_url": "https://raw.githubusercontent.com/vespa-engine/sample-apps/master/Vagrantfile", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/Vagrantfile?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/0cb4b6dbe12404fb0bfa67e8e950287cdad722f4", + "html": "https://github.com/vespa-engine/sample-apps/blob/master/Vagrantfile" + } + }, + { + "name": "_config.yml", + "path": "_config.yml", + "sha": "d0a1af407e12cd80401d55679b5566538a925cc6", + "size": 630, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/_config.yml?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/blob/master/_config.yml", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/d0a1af407e12cd80401d55679b5566538a925cc6", + "download_url": "https://raw.githubusercontent.com/vespa-engine/sample-apps/master/_config.yml", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/_config.yml?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/d0a1af407e12cd80401d55679b5566538a925cc6", + "html": "https://github.com/vespa-engine/sample-apps/blob/master/_config.yml" + } + }, + { + "name": "_plugins-vespafeed", + "path": "_plugins-vespafeed", + "sha": "34a1dec1e26d0905f461a70ebdea5213e8510eea", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/_plugins-vespafeed?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/_plugins-vespafeed", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/34a1dec1e26d0905f461a70ebdea5213e8510eea", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/_plugins-vespafeed?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/34a1dec1e26d0905f461a70ebdea5213e8510eea", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/_plugins-vespafeed" + } + }, + { + "name": "album-recommendation-monitoring", + "path": "album-recommendation-monitoring", + "sha": "f2f816138535aae9c1080c8dca20e544b0a8fbde", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/album-recommendation-monitoring?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/album-recommendation-monitoring", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/f2f816138535aae9c1080c8dca20e544b0a8fbde", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/album-recommendation-monitoring?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/f2f816138535aae9c1080c8dca20e544b0a8fbde", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/album-recommendation-monitoring" + } + }, + { + "name": "album-recommendation-selfhosted", + "path": "album-recommendation-selfhosted", + "sha": "1355199dee65aba689b784a8c92977886e8b0dcb", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/album-recommendation-selfhosted?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/album-recommendation-selfhosted", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/1355199dee65aba689b784a8c92977886e8b0dcb", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/album-recommendation-selfhosted?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/1355199dee65aba689b784a8c92977886e8b0dcb", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/album-recommendation-selfhosted" + } + }, + { + "name": "aws_bootstrap.sh", + "path": "aws_bootstrap.sh", + "sha": "4451c011e7d84d31bda78c6bb64d1b39bf79543c", + "size": 945, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/aws_bootstrap.sh?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/blob/master/aws_bootstrap.sh", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/4451c011e7d84d31bda78c6bb64d1b39bf79543c", + "download_url": "https://raw.githubusercontent.com/vespa-engine/sample-apps/master/aws_bootstrap.sh", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/aws_bootstrap.sh?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/4451c011e7d84d31bda78c6bb64d1b39bf79543c", + "html": "https://github.com/vespa-engine/sample-apps/blob/master/aws_bootstrap.sh" + } + }, + { + "name": "basic-search-on-gke", + "path": "basic-search-on-gke", + "sha": "e3045c5e7e3d5a9604fe5d6e79e28276341e3190", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/basic-search-on-gke?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/basic-search-on-gke", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/e3045c5e7e3d5a9604fe5d6e79e28276341e3190", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/basic-search-on-gke?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/e3045c5e7e3d5a9604fe5d6e79e28276341e3190", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/basic-search-on-gke" + } + }, + { + "name": "boolean-search", + "path": "boolean-search", + "sha": "f698016bfba3492e36251bfdc7af27f3157dcb58", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/boolean-search?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/boolean-search", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/f698016bfba3492e36251bfdc7af27f3157dcb58", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/boolean-search?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/f698016bfba3492e36251bfdc7af27f3157dcb58", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/boolean-search" + } + }, + { + "name": "dense-passage-retrieval-with-ann", + "path": "dense-passage-retrieval-with-ann", + "sha": "3281faf0e968e5dda398f784b0f370b93a94dd7d", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/dense-passage-retrieval-with-ann?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/dense-passage-retrieval-with-ann", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/3281faf0e968e5dda398f784b0f370b93a94dd7d", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/dense-passage-retrieval-with-ann?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/3281faf0e968e5dda398f784b0f370b93a94dd7d", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/dense-passage-retrieval-with-ann" + } + }, + { + "name": "feed_to_vespa.py", + "path": "feed_to_vespa.py", + "sha": "ca825ded90c8c2300b29f6c016b8875c5547f6b6", + "size": 6825, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/feed_to_vespa.py?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/blob/master/feed_to_vespa.py", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/ca825ded90c8c2300b29f6c016b8875c5547f6b6", + "download_url": "https://raw.githubusercontent.com/vespa-engine/sample-apps/master/feed_to_vespa.py", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/feed_to_vespa.py?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/ca825ded90c8c2300b29f6c016b8875c5547f6b6", + "html": "https://github.com/vespa-engine/sample-apps/blob/master/feed_to_vespa.py" + } + }, + { + "name": "generic-request-processing", + "path": "generic-request-processing", + "sha": "ae922ab9efebf6cc9f8ee9726e48693b794eb235", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/generic-request-processing?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/generic-request-processing", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/ae922ab9efebf6cc9f8ee9726e48693b794eb235", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/generic-request-processing?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/ae922ab9efebf6cc9f8ee9726e48693b794eb235", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/generic-request-processing" + } + }, + { + "name": "http-api-using-request-handlers-and-processors", + "path": "http-api-using-request-handlers-and-processors", + "sha": "b8403feb85fe2a113114a2bd086057aec0538e46", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/http-api-using-request-handlers-and-processors?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/http-api-using-request-handlers-and-processors", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/b8403feb85fe2a113114a2bd086057aec0538e46", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/http-api-using-request-handlers-and-processors?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/b8403feb85fe2a113114a2bd086057aec0538e46", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/http-api-using-request-handlers-and-processors" + } + }, + { + "name": "incremental-search", + "path": "incremental-search", + "sha": "6d38eb194e4a8f6565a66591758a17bf8e6459e9", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/incremental-search?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/incremental-search", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/6d38eb194e4a8f6565a66591758a17bf8e6459e9", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/incremental-search?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/6d38eb194e4a8f6565a66591758a17bf8e6459e9", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/incremental-search" + } + }, + { + "name": "model-evaluation", + "path": "model-evaluation", + "sha": "5f782af796cd13ebfb0c3e2d4ab074bc33e41913", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/model-evaluation?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/model-evaluation", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/5f782af796cd13ebfb0c3e2d4ab074bc33e41913", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/model-evaluation?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/5f782af796cd13ebfb0c3e2d4ab074bc33e41913", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/model-evaluation" + } + }, + { + "name": "msmarco-ranking", + "path": "msmarco-ranking", + "sha": "44af5a222f523e0c1aa92fc11dda4cbe095ca6b1", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/msmarco-ranking?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/msmarco-ranking", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/44af5a222f523e0c1aa92fc11dda4cbe095ca6b1", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/msmarco-ranking?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/44af5a222f523e0c1aa92fc11dda4cbe095ca6b1", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/msmarco-ranking" + } + }, + { + "name": "multiple-bundles-lib", + "path": "multiple-bundles-lib", + "sha": "8604cc153faee34c538a82b275e09f743f83216b", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/multiple-bundles-lib?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/multiple-bundles-lib", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/8604cc153faee34c538a82b275e09f743f83216b", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/multiple-bundles-lib?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/8604cc153faee34c538a82b275e09f743f83216b", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/multiple-bundles-lib" + } + }, + { + "name": "multiple-bundles", + "path": "multiple-bundles", + "sha": "1cc37ba975795212e19801a1f2c1dd27abfbec77", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/multiple-bundles?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/multiple-bundles", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/1cc37ba975795212e19801a1f2c1dd27abfbec77", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/multiple-bundles?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/1cc37ba975795212e19801a1f2c1dd27abfbec77", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/multiple-bundles" + } + }, + { + "name": "news", + "path": "news", + "sha": "aed7b1fde19b139781fb5df4d7feb3e8ec1db603", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/news", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/aed7b1fde19b139781fb5df4d7feb3e8ec1db603", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/aed7b1fde19b139781fb5df4d7feb3e8ec1db603", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/news" + } + }, + { + "name": "operations", + "path": "operations", + "sha": "d73d99d8a7b2f496ff1e43da6b3f87d612227232", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/operations?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/operations", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/d73d99d8a7b2f496ff1e43da6b3f87d612227232", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/operations?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/d73d99d8a7b2f496ff1e43da6b3f87d612227232", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/operations" + } + }, + { + "name": "part-purchases-demo", + "path": "part-purchases-demo", + "sha": "c1f64f68d2abe8b207da901e6ccb9277971167dc", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/part-purchases-demo?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/part-purchases-demo", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/c1f64f68d2abe8b207da901e6ccb9277971167dc", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/part-purchases-demo?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/c1f64f68d2abe8b207da901e6ccb9277971167dc", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/part-purchases-demo" + } + }, + { + "name": "pom.xml", + "path": "pom.xml", + "sha": "191e06595d1db24d864833b333539a083d8a5f30", + "size": 1839, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/pom.xml?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/blob/master/pom.xml", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/191e06595d1db24d864833b333539a083d8a5f30", + "download_url": "https://raw.githubusercontent.com/vespa-engine/sample-apps/master/pom.xml", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/pom.xml?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/191e06595d1db24d864833b333539a083d8a5f30", + "html": "https://github.com/vespa-engine/sample-apps/blob/master/pom.xml" + } + }, + { + "name": "screwdriver.yaml", + "path": "screwdriver.yaml", + "sha": "fd9460121b230498baed2b1760b36e2f1a067fd7", + "size": 2771, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/screwdriver.yaml?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/blob/master/screwdriver.yaml", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/fd9460121b230498baed2b1760b36e2f1a067fd7", + "download_url": "https://raw.githubusercontent.com/vespa-engine/sample-apps/master/screwdriver.yaml", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/screwdriver.yaml?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/fd9460121b230498baed2b1760b36e2f1a067fd7", + "html": "https://github.com/vespa-engine/sample-apps/blob/master/screwdriver.yaml" + } + }, + { + "name": "secure-vespa-with-mtls", + "path": "secure-vespa-with-mtls", + "sha": "c6321c984bc4965f274d5f773af241935edb21a2", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/secure-vespa-with-mtls?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/secure-vespa-with-mtls", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/c6321c984bc4965f274d5f773af241935edb21a2", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/secure-vespa-with-mtls?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/c6321c984bc4965f274d5f773af241935edb21a2", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/secure-vespa-with-mtls" + } + }, + { + "name": "semantic-qa-retrieval", + "path": "semantic-qa-retrieval", + "sha": "67bc047714c2c76e1318eb8faa5b208747b62d5e", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/semantic-qa-retrieval?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/semantic-qa-retrieval", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/67bc047714c2c76e1318eb8faa5b208747b62d5e", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/semantic-qa-retrieval?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/67bc047714c2c76e1318eb8faa5b208747b62d5e", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/semantic-qa-retrieval" + } + }, + { + "name": "tensor-playground", + "path": "tensor-playground", + "sha": "090954a12a47a98250b729b0c01bf804577cb7ad", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/tensor-playground?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/tensor-playground", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/090954a12a47a98250b729b0c01bf804577cb7ad", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/tensor-playground?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/090954a12a47a98250b729b0c01bf804577cb7ad", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/tensor-playground" + } + }, + { + "name": "test", + "path": "test", + "sha": "0948c9e369a9734856c5d55b11c9fc7681ca398b", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/test?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/test", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/0948c9e369a9734856c5d55b11c9fc7681ca398b", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/test?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/0948c9e369a9734856c5d55b11c9fc7681ca398b", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/test" + } + }, + { + "name": "text-search", + "path": "text-search", + "sha": "7c3eb5ebfa59f0fdc7f1d3cd9601f238c4c1461a", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/text-search?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/text-search", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/7c3eb5ebfa59f0fdc7f1d3cd9601f238c4c1461a", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/text-search?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/7c3eb5ebfa59f0fdc7f1d3cd9601f238c4c1461a", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/text-search" + } + }, + { + "name": "transformers", + "path": "transformers", + "sha": "43a907697bbb4a6baa673a3005bc009adff16cf3", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/transformers?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/transformers", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/43a907697bbb4a6baa673a3005bc009adff16cf3", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/transformers?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/43a907697bbb4a6baa673a3005bc009adff16cf3", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/transformers" + } + }, + { + "name": "use-case-shopping", + "path": "use-case-shopping", + "sha": "7083dda1a5d3ad4ea5b349e67cdd2ab9b57dc305", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/use-case-shopping?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/use-case-shopping", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/7083dda1a5d3ad4ea5b349e67cdd2ab9b57dc305", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/use-case-shopping?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/7083dda1a5d3ad4ea5b349e67cdd2ab9b57dc305", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/use-case-shopping" + } + }, + { + "name": "vespa-chinese-linguistics", + "path": "vespa-chinese-linguistics", + "sha": "de804dec91603fc905c82dbb982ca9c2dbeea99c", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/vespa-chinese-linguistics?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/vespa-chinese-linguistics", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/de804dec91603fc905c82dbb982ca9c2dbeea99c", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/vespa-chinese-linguistics?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/de804dec91603fc905c82dbb982ca9c2dbeea99c", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/vespa-chinese-linguistics" + } + }, + { + "name": "vespa-cloud", + "path": "vespa-cloud", + "sha": "1f4ae8043e03f1d31bd361f1635cf3fe7dfe7c0b", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/vespa-cloud?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/vespa-cloud", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/1f4ae8043e03f1d31bd361f1635cf3fe7dfe7c0b", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/vespa-cloud?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/1f4ae8043e03f1d31bd361f1635cf3fe7dfe7c0b", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/vespa-cloud" + } + } +] diff --git a/client/go/internal/cli/cmd/testdata/sample-apps-master.zip b/client/go/internal/cli/cmd/testdata/sample-apps-master.zip Binary files differnew file mode 100644 index 00000000000..c8fb40af713 --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/sample-apps-master.zip diff --git a/client/go/internal/cli/cmd/testdata/sample-apps-news.json b/client/go/internal/cli/cmd/testdata/sample-apps-news.json new file mode 100644 index 00000000000..495f2cf1294 --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/sample-apps-news.json @@ -0,0 +1,178 @@ +[ + { + "name": "README.md", + "path": "news/README.md", + "sha": "faaf9e19a01ae471d7a53ba9ed727bd792edf77c", + "size": 193, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news/README.md?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/blob/master/news/README.md", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/faaf9e19a01ae471d7a53ba9ed727bd792edf77c", + "download_url": "https://raw.githubusercontent.com/vespa-engine/sample-apps/master/news/README.md", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news/README.md?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/faaf9e19a01ae471d7a53ba9ed727bd792edf77c", + "html": "https://github.com/vespa-engine/sample-apps/blob/master/news/README.md" + } + }, + { + "name": "app-1-getting-started", + "path": "news/app-1-getting-started", + "sha": "beba7dde150de79caf412a90ccf10177aef3af92", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news/app-1-getting-started?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/news/app-1-getting-started", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/beba7dde150de79caf412a90ccf10177aef3af92", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news/app-1-getting-started?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/beba7dde150de79caf412a90ccf10177aef3af92", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/news/app-1-getting-started" + } + }, + { + "name": "app-2-feed-and-query", + "path": "news/app-2-feed-and-query", + "sha": "508dff55513ec03db25765b1022b85f5d5b225c3", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news/app-2-feed-and-query?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/news/app-2-feed-and-query", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/508dff55513ec03db25765b1022b85f5d5b225c3", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news/app-2-feed-and-query?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/508dff55513ec03db25765b1022b85f5d5b225c3", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/news/app-2-feed-and-query" + } + }, + { + "name": "app-3-searching", + "path": "news/app-3-searching", + "sha": "431c96c3a93e0ac988acb6349456b1b423fb6ff8", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news/app-3-searching?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/news/app-3-searching", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/431c96c3a93e0ac988acb6349456b1b423fb6ff8", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news/app-3-searching?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/431c96c3a93e0ac988acb6349456b1b423fb6ff8", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/news/app-3-searching" + } + }, + { + "name": "app-5-recommendation", + "path": "news/app-5-recommendation", + "sha": "fdbc92176cf8ee10732f3f14726ab9fba41652e0", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news/app-5-recommendation?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/news/app-5-recommendation", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/fdbc92176cf8ee10732f3f14726ab9fba41652e0", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news/app-5-recommendation?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/fdbc92176cf8ee10732f3f14726ab9fba41652e0", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/news/app-5-recommendation" + } + }, + { + "name": "app-6-recommendation-with-searchers", + "path": "news/app-6-recommendation-with-searchers", + "sha": "b636cb11c3c3c2f5334b04e151101c100d22c375", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news/app-6-recommendation-with-searchers?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/news/app-6-recommendation-with-searchers", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/b636cb11c3c3c2f5334b04e151101c100d22c375", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news/app-6-recommendation-with-searchers?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/b636cb11c3c3c2f5334b04e151101c100d22c375", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/news/app-6-recommendation-with-searchers" + } + }, + { + "name": "app-7-parent-child", + "path": "news/app-7-parent-child", + "sha": "56760f75354c43c7cdc387758f6b68659c10087b", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news/app-7-parent-child?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/news/app-7-parent-child", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/56760f75354c43c7cdc387758f6b68659c10087b", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news/app-7-parent-child?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/56760f75354c43c7cdc387758f6b68659c10087b", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/news/app-7-parent-child" + } + }, + { + "name": "bin", + "path": "news/bin", + "sha": "7c755f24e4d885d3f6677ec593e46c7d64c377f8", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news/bin?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/news/bin", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/7c755f24e4d885d3f6677ec593e46c7d64c377f8", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news/bin?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/7c755f24e4d885d3f6677ec593e46c7d64c377f8", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/news/bin" + } + }, + { + "name": "doc.json", + "path": "news/doc.json", + "sha": "4f7cad1c86d48ad9b1470c19bcec0a9c136dee11", + "size": 72, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news/doc.json?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/blob/master/news/doc.json", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/4f7cad1c86d48ad9b1470c19bcec0a9c136dee11", + "download_url": "https://raw.githubusercontent.com/vespa-engine/sample-apps/master/news/doc.json", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news/doc.json?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/4f7cad1c86d48ad9b1470c19bcec0a9c136dee11", + "html": "https://github.com/vespa-engine/sample-apps/blob/master/news/doc.json" + } + }, + { + "name": "requirements.txt", + "path": "news/requirements.txt", + "sha": "21fe4a3bc4ed1fcc2797c8a9f4188e48fed6d4e7", + "size": 37, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news/requirements.txt?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/blob/master/news/requirements.txt", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/21fe4a3bc4ed1fcc2797c8a9f4188e48fed6d4e7", + "download_url": "https://raw.githubusercontent.com/vespa-engine/sample-apps/master/news/requirements.txt", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news/requirements.txt?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/21fe4a3bc4ed1fcc2797c8a9f4188e48fed6d4e7", + "html": "https://github.com/vespa-engine/sample-apps/blob/master/news/requirements.txt" + } + }, + { + "name": "src", + "path": "news/src", + "sha": "106a0eca050c215f3b6a919640d6b4449d0c0aa5", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news/src?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/news/src", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/106a0eca050c215f3b6a919640d6b4449d0c0aa5", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/news/src?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/106a0eca050c215f3b6a919640d6b4449d0c0aa5", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/news/src" + } + } +] diff --git a/client/go/internal/cli/cmd/testdata/sample-apps-operations.json b/client/go/internal/cli/cmd/testdata/sample-apps-operations.json new file mode 100644 index 00000000000..9dc4fb39ddb --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/sample-apps-operations.json @@ -0,0 +1,18 @@ +[ + { + "name": "multinode", + "path": "operations/multinode", + "sha": "faa2b5bf9fa6fc7314708341e2ad91211986d681", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/operations/multinode?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/operations/multinode", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/faa2b5bf9fa6fc7314708341e2ad91211986d681", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/operations/multinode?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/faa2b5bf9fa6fc7314708341e2ad91211986d681", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/operations/multinode" + } + } +] diff --git a/client/go/internal/cli/cmd/testdata/sample-apps-vespa-cloud.json b/client/go/internal/cli/cmd/testdata/sample-apps-vespa-cloud.json new file mode 100644 index 00000000000..3aa932c348f --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/sample-apps-vespa-cloud.json @@ -0,0 +1,130 @@ +[ + { + "name": "README.md", + "path": "vespa-cloud/README.md", + "sha": "6405553e7184306476d1e13001aa19558f925158", + "size": 3129, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/vespa-cloud/README.md?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/blob/master/vespa-cloud/README.md", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/6405553e7184306476d1e13001aa19558f925158", + "download_url": "https://raw.githubusercontent.com/vespa-engine/sample-apps/master/vespa-cloud/README.md", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/vespa-cloud/README.md?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/blobs/6405553e7184306476d1e13001aa19558f925158", + "html": "https://github.com/vespa-engine/sample-apps/blob/master/vespa-cloud/README.md" + } + }, + { + "name": "album-recommendation-docproc", + "path": "vespa-cloud/album-recommendation-docproc", + "sha": "e77b300ad64b1c553d066c7840cbb8f555df91cd", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/vespa-cloud/album-recommendation-docproc?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/vespa-cloud/album-recommendation-docproc", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/e77b300ad64b1c553d066c7840cbb8f555df91cd", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/vespa-cloud/album-recommendation-docproc?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/e77b300ad64b1c553d066c7840cbb8f555df91cd", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/vespa-cloud/album-recommendation-docproc" + } + }, + { + "name": "album-recommendation-prod", + "path": "vespa-cloud/album-recommendation-prod", + "sha": "0944ec03009e38ff1c12be835ca6a0600e6aee93", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/vespa-cloud/album-recommendation-prod?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/vespa-cloud/album-recommendation-prod", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/0944ec03009e38ff1c12be835ca6a0600e6aee93", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/vespa-cloud/album-recommendation-prod?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/0944ec03009e38ff1c12be835ca6a0600e6aee93", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/vespa-cloud/album-recommendation-prod" + } + }, + { + "name": "album-recommendation-searcher", + "path": "vespa-cloud/album-recommendation-searcher", + "sha": "fd1027d9cd51a2aac9f2220047b5a748ef75d2c1", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/vespa-cloud/album-recommendation-searcher?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/vespa-cloud/album-recommendation-searcher", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/fd1027d9cd51a2aac9f2220047b5a748ef75d2c1", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/vespa-cloud/album-recommendation-searcher?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/fd1027d9cd51a2aac9f2220047b5a748ef75d2c1", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/vespa-cloud/album-recommendation-searcher" + } + }, + { + "name": "album-recommendation", + "path": "vespa-cloud/album-recommendation", + "sha": "fdb9ffd4dff104a8a603d5ce0ead1bed180f8b80", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/vespa-cloud/album-recommendation?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/vespa-cloud/album-recommendation", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/fdb9ffd4dff104a8a603d5ce0ead1bed180f8b80", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/vespa-cloud/album-recommendation?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/fdb9ffd4dff104a8a603d5ce0ead1bed180f8b80", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/vespa-cloud/album-recommendation" + } + }, + { + "name": "cord-19-search", + "path": "vespa-cloud/cord-19-search", + "sha": "29a28e0e0455d2bcd5c7bcdcd3c8759455f6bc71", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/vespa-cloud/cord-19-search?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/vespa-cloud/cord-19-search", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/29a28e0e0455d2bcd5c7bcdcd3c8759455f6bc71", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/vespa-cloud/cord-19-search?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/29a28e0e0455d2bcd5c7bcdcd3c8759455f6bc71", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/vespa-cloud/cord-19-search" + } + }, + { + "name": "joins", + "path": "vespa-cloud/joins", + "sha": "1f3272bfa1c4dd6cb9d26b662bf48de3c05aef04", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/vespa-cloud/joins?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/vespa-cloud/joins", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/1f3272bfa1c4dd6cb9d26b662bf48de3c05aef04", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/vespa-cloud/joins?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/1f3272bfa1c4dd6cb9d26b662bf48de3c05aef04", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/vespa-cloud/joins" + } + }, + { + "name": "vespa-documentation-search", + "path": "vespa-cloud/vespa-documentation-search", + "sha": "9c46d6eb0eef9ce95688d96ef603a5143c3c46c0", + "size": 0, + "url": "https://api.github.com/repos/vespa-engine/sample-apps/contents/vespa-cloud/vespa-documentation-search?ref=master", + "html_url": "https://github.com/vespa-engine/sample-apps/tree/master/vespa-cloud/vespa-documentation-search", + "git_url": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/9c46d6eb0eef9ce95688d96ef603a5143c3c46c0", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/vespa-engine/sample-apps/contents/vespa-cloud/vespa-documentation-search?ref=master", + "git": "https://api.github.com/repos/vespa-engine/sample-apps/git/trees/9c46d6eb0eef9ce95688d96ef603a5143c3c46c0", + "html": "https://github.com/vespa-engine/sample-apps/tree/master/vespa-cloud/vespa-documentation-search" + } + } +] diff --git a/client/go/internal/cli/cmd/testdata/tests/body.json b/client/go/internal/cli/cmd/testdata/tests/body.json new file mode 100644 index 00000000000..767330b1a2d --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/tests/body.json @@ -0,0 +1,12 @@ +{ + "root": { + "id": "toplevel", + "coverage": { + "full": true + }, + "fields": { + "totalCount" : 1 + }, + "children": [{}] + } +}
\ No newline at end of file diff --git a/client/go/internal/cli/cmd/testdata/tests/expected-suite.out b/client/go/internal/cli/cmd/testdata/tests/expected-suite.out new file mode 100644 index 00000000000..df916f50a95 --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/tests/expected-suite.out @@ -0,0 +1,385 @@ +My test: .... OK +wrong-bool-value.json: failed +Step 1: +Unexpected value at /root/coverage/full +Expected: false +Actual: true +Requested: GET at http://127.0.0.1:8080/search/ +Response: +{ + "root": { + "children": [ + { + "fields": { + "artist": "Foo Fighters", + "documentid": "id:test:music::doc", + "sddocname": "music" + }, + "id": "id:test:music::doc", + "relevance": 0.38186238359951247, + "source": "music" + } + ], + "coverage": { + "coverage": 100, + "documents": 1, + "full": true, + "nodes": 1, + "results": 1, + "resultsFull": 1 + }, + "fields": { + "totalCount": 1 + }, + "id": "toplevel", + "relevance": 1 + }, + "timing": { + "querytime": 0.003, + "searchtime": 0.004, + "summaryfetchtime": 0 + } +} + +wrong-code.json: failed +Step 1: +Unexpected status code +Expected: 123 +Actual: 200 +Requested: GET at http://127.0.0.1:8080/search/?foo=%2F +Response: +{ + "root": { + "children": [ + { + "fields": { + "artist": "Foo Fighters", + "documentid": "id:test:music::doc", + "sddocname": "music" + }, + "id": "id:test:music::doc", + "relevance": 0.38186238359951247, + "source": "music" + } + ], + "coverage": { + "coverage": 100, + "documents": 1, + "full": true, + "nodes": 1, + "results": 1, + "resultsFull": 1 + }, + "fields": { + "totalCount": 1 + }, + "id": "toplevel", + "relevance": 1 + }, + "timing": { + "querytime": 0.003, + "searchtime": 0.004, + "summaryfetchtime": 0 + } +} + +wrong-element-count.json: failed +Step 1: +Unexpected number of elements at /root/children +Expected: 0 +Actual: 1 +Requested: GET at http://127.0.0.1:8080/search/ +Response: +{ + "root": { + "children": [ + { + "fields": { + "artist": "Foo Fighters", + "documentid": "id:test:music::doc", + "sddocname": "music" + }, + "id": "id:test:music::doc", + "relevance": 0.38186238359951247, + "source": "music" + } + ], + "coverage": { + "coverage": 100, + "documents": 1, + "full": true, + "nodes": 1, + "results": 1, + "resultsFull": 1 + }, + "fields": { + "totalCount": 1 + }, + "id": "toplevel", + "relevance": 1 + }, + "timing": { + "querytime": 0.003, + "searchtime": 0.004, + "summaryfetchtime": 0 + } +} + +wrong-field-name.json: failed +Step 1: +Missing expected field at /root/fields/totalCountDracula +Requested: GET at http://127.0.0.1:8080/search/ +Response: +{ + "root": { + "children": [ + { + "fields": { + "artist": "Foo Fighters", + "documentid": "id:test:music::doc", + "sddocname": "music" + }, + "id": "id:test:music::doc", + "relevance": 0.38186238359951247, + "source": "music" + } + ], + "coverage": { + "coverage": 100, + "documents": 1, + "full": true, + "nodes": 1, + "results": 1, + "resultsFull": 1 + }, + "fields": { + "totalCount": 1 + }, + "id": "toplevel", + "relevance": 1 + }, + "timing": { + "querytime": 0.003, + "searchtime": 0.004, + "summaryfetchtime": 0 + } +} + +wrong-float-value.json: failed +Step 1: +Unexpected value at /root/children/0/relevance +Expected: 0.381862373599 +Actual: 0.38186238359951247 +Requested: GET at http://127.0.0.1:8080/search/ +Response: +{ + "root": { + "children": [ + { + "fields": { + "artist": "Foo Fighters", + "documentid": "id:test:music::doc", + "sddocname": "music" + }, + "id": "id:test:music::doc", + "relevance": 0.38186238359951247, + "source": "music" + } + ], + "coverage": { + "coverage": 100, + "documents": 1, + "full": true, + "nodes": 1, + "results": 1, + "resultsFull": 1 + }, + "fields": { + "totalCount": 1 + }, + "id": "toplevel", + "relevance": 1 + }, + "timing": { + "querytime": 0.003, + "searchtime": 0.004, + "summaryfetchtime": 0 + } +} + +wrong-int-value.json: failed +Step 1: +Unexpected value at /root/fields/totalCount +Expected: 2 +Actual: 1 +Requested: GET at http://127.0.0.1:8080/search/ +Response: +{ + "root": { + "children": [ + { + "fields": { + "artist": "Foo Fighters", + "documentid": "id:test:music::doc", + "sddocname": "music" + }, + "id": "id:test:music::doc", + "relevance": 0.38186238359951247, + "source": "music" + } + ], + "coverage": { + "coverage": 100, + "documents": 1, + "full": true, + "nodes": 1, + "results": 1, + "resultsFull": 1 + }, + "fields": { + "totalCount": 1 + }, + "id": "toplevel", + "relevance": 1 + }, + "timing": { + "querytime": 0.003, + "searchtime": 0.004, + "summaryfetchtime": 0 + } +} + +wrong-null-value.json: failed +Step 1: +Missing expected field at /boot +Requested: GET at http://127.0.0.1:8080/search/ +Response: +{ + "root": { + "children": [ + { + "fields": { + "artist": "Foo Fighters", + "documentid": "id:test:music::doc", + "sddocname": "music" + }, + "id": "id:test:music::doc", + "relevance": 0.38186238359951247, + "source": "music" + } + ], + "coverage": { + "coverage": 100, + "documents": 1, + "full": true, + "nodes": 1, + "results": 1, + "resultsFull": 1 + }, + "fields": { + "totalCount": 1 + }, + "id": "toplevel", + "relevance": 1 + }, + "timing": { + "querytime": 0.003, + "searchtime": 0.004, + "summaryfetchtime": 0 + } +} + +wrong-string-value.json: failed +Step 1: +Unexpected value at /root/children/0/fields/artist +Expected: "Boo Fighters" +Actual: "Foo Fighters" +Requested: GET at http://127.0.0.1:8080/search/ +Response: +{ + "root": { + "children": [ + { + "fields": { + "artist": "Foo Fighters", + "documentid": "id:test:music::doc", + "sddocname": "music" + }, + "id": "id:test:music::doc", + "relevance": 0.38186238359951247, + "source": "music" + } + ], + "coverage": { + "coverage": 100, + "documents": 1, + "full": true, + "nodes": 1, + "results": 1, + "resultsFull": 1 + }, + "fields": { + "totalCount": 1 + }, + "id": "toplevel", + "relevance": 1 + }, + "timing": { + "querytime": 0.003, + "searchtime": 0.004, + "summaryfetchtime": 0 + } +} + +wrong-type.json: failed +Step 1: +Unexpected type at /root/fields/totalCount +Expected: "1" +Actual: 1 +Requested: GET at http://127.0.0.1:8080/search/ +Response: +{ + "root": { + "children": [ + { + "fields": { + "artist": "Foo Fighters", + "documentid": "id:test:music::doc", + "sddocname": "music" + }, + "id": "id:test:music::doc", + "relevance": 0.38186238359951247, + "source": "music" + } + ], + "coverage": { + "coverage": 100, + "documents": 1, + "full": true, + "nodes": 1, + "results": 1, + "resultsFull": 1 + }, + "fields": { + "totalCount": 1 + }, + "id": "toplevel", + "relevance": 1 + }, + "timing": { + "querytime": 0.003, + "searchtime": 0.004, + "summaryfetchtime": 0 + } +} + +Failure: 9 of 10 tests failed: +wrong-bool-value.json: Step 1: Unexpected value at /root/coverage/full: true +wrong-code.json: Step 1: Unexpected status code: 200 +wrong-element-count.json: Step 1: Unexpected number of elements at /root/children: 1 +wrong-field-name.json: Step 1: Missing expected field at /root/fields/totalCountDracula +wrong-float-value.json: Step 1: Unexpected value at /root/children/0/relevance: 0.38186238359951247 +wrong-int-value.json: Step 1: Unexpected value at /root/fields/totalCount: 1 +wrong-null-value.json: Step 1: Missing expected field at /boot +wrong-string-value.json: Step 1: Unexpected value at /root/children/0/fields/artist: "Foo Fighters" +wrong-type.json: Step 1: Unexpected type at /root/fields/totalCount: 1 diff --git a/client/go/internal/cli/cmd/testdata/tests/expected.out b/client/go/internal/cli/cmd/testdata/tests/expected.out new file mode 100644 index 00000000000..2ca35fe6a37 --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/tests/expected.out @@ -0,0 +1,3 @@ +My test: .... OK + +Success: 1 test OK diff --git a/client/go/internal/cli/cmd/testdata/tests/production-test/external.json b/client/go/internal/cli/cmd/testdata/tests/production-test/external.json new file mode 100644 index 00000000000..af288bc8b1b --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/tests/production-test/external.json @@ -0,0 +1,9 @@ +{ + "steps": [ + { + "request": { + "uri": "https://my.service:123/path?query=wohoo" + } + } + ] +}
\ No newline at end of file diff --git a/client/go/internal/cli/cmd/testdata/tests/production-test/illegal-reference.json b/client/go/internal/cli/cmd/testdata/tests/production-test/illegal-reference.json new file mode 100644 index 00000000000..ced4a86dd6c --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/tests/production-test/illegal-reference.json @@ -0,0 +1,16 @@ +{ + "steps": [ + { + "request": { + "uri": "https://domain.tld", + "body": "foo/../../../empty.json" + } + }, + { + "request": { + "uri": "https://domain.tld", + "body": "foo/../../../../this-is-not-ok.json" + } + } + ] +}
\ No newline at end of file diff --git a/client/go/internal/cli/cmd/testdata/tests/production-test/illegal-uri.json b/client/go/internal/cli/cmd/testdata/tests/production-test/illegal-uri.json new file mode 100644 index 00000000000..fe4fadfa93d --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/tests/production-test/illegal-uri.json @@ -0,0 +1,14 @@ +{ + "steps": [ + { + "request": { + "uri": "https://domain.tld/my-api" + } + }, + { + "request": { + "uri": "/my-api" + } + } + ] +}
\ No newline at end of file diff --git a/client/go/internal/cli/cmd/testdata/tests/response.json b/client/go/internal/cli/cmd/testdata/tests/response.json new file mode 100644 index 00000000000..48368b935a8 --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/tests/response.json @@ -0,0 +1,34 @@ +{ + "root": { + "children": [ + { + "fields": { + "artist": "Foo Fighters", + "documentid": "id:test:music::doc", + "sddocname": "music" + }, + "id": "id:test:music::doc", + "relevance": 0.38186238359951247, + "source": "music" + } + ], + "coverage": { + "coverage": 100, + "documents": 1, + "full": true, + "nodes": 1, + "results": 1, + "resultsFull": 1 + }, + "fields": { + "totalCount": 1 + }, + "id": "toplevel", + "relevance": 1 + }, + "timing": { + "querytime": 0.003, + "searchtime": 0.004, + "summaryfetchtime": 0 + } +}
\ No newline at end of file diff --git a/client/go/internal/cli/cmd/testdata/tests/staging-test/not-json b/client/go/internal/cli/cmd/testdata/tests/staging-test/not-json new file mode 100644 index 00000000000..b6fc4c620b6 --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/tests/staging-test/not-json @@ -0,0 +1 @@ +hello
\ No newline at end of file diff --git a/client/go/internal/cli/cmd/testdata/tests/system-test/foo/body.json b/client/go/internal/cli/cmd/testdata/tests/system-test/foo/body.json new file mode 100644 index 00000000000..0bbf626eafe --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/tests/system-test/foo/body.json @@ -0,0 +1,5 @@ +{ + "fields": { + "artist": "Foo Fighters" + } +}
\ No newline at end of file diff --git a/client/go/internal/cli/cmd/testdata/tests/system-test/foo/query.json b/client/go/internal/cli/cmd/testdata/tests/system-test/foo/query.json new file mode 100644 index 00000000000..25b8c5b0039 --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/tests/system-test/foo/query.json @@ -0,0 +1,3 @@ +{ + "query": "artist: foo" +} diff --git a/client/go/internal/cli/cmd/testdata/tests/system-test/test.json b/client/go/internal/cli/cmd/testdata/tests/system-test/test.json new file mode 100644 index 00000000000..2e327b5e5df --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/tests/system-test/test.json @@ -0,0 +1,65 @@ +{ + "name": "My test", + "defaults": { + "cluster": "container", + "parameters": { + "timeout": "3.4s" + } + }, + "steps": [ + { + "name": "feed music", + "request": { + "method": "POST", + "body": "foo/body.json", + "uri": "/document/v1/test/music/docid/doc" + } + }, + { + "name": "re-feed music", + "request": { + "method": "POST", + "body": { + "fields": { + "artist": "Foo Fighters" + } + }, + "uri": "/document/v1/test/music/docid/doc" + } + }, + { + "name": "query for foo", + "request": { + "uri": "/search/?presentation.timing=true", + "parameters": { + "query": "artist: foo" + } + }, + "response": { + "code": 200, + "body": "../body.json" + } + }, + { + "name": "query for foo again", + "request": { + "uri": "/search/?presentation.timing=true", + "parameters": "foo/query.json" + }, + "response": { + "body": { + "root": { + "children": [ + { + "fields": { + "artist": "Foo Fighters" + }, + "relevance": 0.381862383599 + } + ] + } + } + } + } + ] +} diff --git a/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-bool-value.json b/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-bool-value.json new file mode 100644 index 00000000000..c594a206347 --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-bool-value.json @@ -0,0 +1,15 @@ +{ + "steps": [ + { + "response": { + "body": { + "root": { + "coverage": { + "full": false + } + } + } + } + } + ] +} diff --git a/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-code.json b/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-code.json new file mode 100644 index 00000000000..c325054faa1 --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-code.json @@ -0,0 +1,14 @@ +{ + "steps": [ + { + "request": { + "parameters": { + "foo": "/" + } + }, + "response": { + "code": 123 + } + } + ] +} diff --git a/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-element-count.json b/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-element-count.json new file mode 100644 index 00000000000..a772af67a78 --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-element-count.json @@ -0,0 +1,13 @@ +{ + "steps": [ + { + "response": { + "body": { + "root": { + "children": [] + } + } + } + } + ] +} diff --git a/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-field-name.json b/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-field-name.json new file mode 100644 index 00000000000..6ce3d055584 --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-field-name.json @@ -0,0 +1,15 @@ +{ + "steps": [ + { + "response": { + "body": { + "root": { + "fields": { + "totalCountDracula" : 1 + } + } + } + } + } + ] +} diff --git a/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-float-value.json b/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-float-value.json new file mode 100644 index 00000000000..6a1b221a91a --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-float-value.json @@ -0,0 +1,17 @@ +{ + "steps": [ + { + "response": { + "body": { + "root": { + "children": [ + { + "relevance": 0.381862373599 + } + ] + } + } + } + } + ] +} diff --git a/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-int-value.json b/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-int-value.json new file mode 100644 index 00000000000..d61a8b002c2 --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-int-value.json @@ -0,0 +1,15 @@ +{ + "steps": [ + { + "response": { + "body": { + "root": { + "fields": { + "totalCount" : 2 + } + } + } + } + } + ] +} diff --git a/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-null-value.json b/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-null-value.json new file mode 100644 index 00000000000..ea78357c99e --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-null-value.json @@ -0,0 +1,11 @@ +{ + "steps": [ + { + "response": { + "body": { + "boot": null + } + } + } + ] +} diff --git a/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-string-value.json b/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-string-value.json new file mode 100644 index 00000000000..5f56ebaab6d --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-string-value.json @@ -0,0 +1,19 @@ +{ + "steps": [ + { + "response": { + "body": { + "root": { + "children": [ + { + "fields": { + "artist": "Boo Fighters" + } + } + ] + } + } + } + } + ] +} diff --git a/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-type.json b/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-type.json new file mode 100644 index 00000000000..6be28ff68ff --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/tests/system-test/wrong-type.json @@ -0,0 +1,15 @@ +{ + "steps": [ + { + "response": { + "body": { + "root": { + "fields": { + "totalCount" : "1" + } + } + } + } + } + ] +} diff --git a/client/go/internal/cli/cmd/testutil_test.go b/client/go/internal/cli/cmd/testutil_test.go new file mode 100644 index 00000000000..6eade6edd86 --- /dev/null +++ b/client/go/internal/cli/cmd/testutil_test.go @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package cmd + +import ( + "bytes" + "path/filepath" + "testing" + + "github.com/vespa-engine/vespa/client/go/internal/mock" +) + +func newTestCLI(t *testing.T, envVars ...string) (*CLI, *bytes.Buffer, *bytes.Buffer) { + t.Helper() + homeDir := filepath.Join(t.TempDir(), ".vespa") + cacheDir := filepath.Join(t.TempDir(), ".cache", "vespa") + env := []string{"VESPA_CLI_HOME=" + homeDir, "VESPA_CLI_CACHE_DIR=" + cacheDir} + env = append(env, envVars...) + var ( + stdout bytes.Buffer + stderr bytes.Buffer + ) + cli, err := New(&stdout, &stderr, env) + if err != nil { + t.Fatal(err) + } + cli.httpClient = &mock.HTTPClient{} + cli.exec = &mock.Exec{} + return cli, &stdout, &stderr +} diff --git a/client/go/internal/cli/cmd/version.go b/client/go/internal/cli/cmd/version.go new file mode 100644 index 00000000000..864c0668eda --- /dev/null +++ b/client/go/internal/cli/cmd/version.go @@ -0,0 +1,126 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package cmd + +import ( + "encoding/json" + "log" + "net/http" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "time" + + "github.com/fatih/color" + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/cli/build" + "github.com/vespa-engine/vespa/client/go/internal/version" +) + +func newVersionCmd(cli *CLI) *cobra.Command { + var skipVersionCheck bool + cmd := &cobra.Command{ + Use: "version", + Short: "Show current version and check for updates", + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + log.Printf("Vespa CLI version %s compiled with %v on %v/%v", build.Version, runtime.Version(), runtime.GOOS, runtime.GOARCH) + if !skipVersionCheck && cli.isTerminal() { + return checkVersion(cli) + } + return nil + }, + } + cmd.Flags().BoolVarP(&skipVersionCheck, "no-check", "n", false, "Do not check if a new version is available") + return cmd +} + +func checkVersion(cli *CLI) error { + current, err := version.Parse(build.Version) + if err != nil { + return err + } + latest, err := latestRelease(cli) + if err != nil { + return err + } + if !current.Less(latest.Version) { + return nil + } + usingHomebrew := usingHomebrew(cli) + if usingHomebrew && latest.isRecent() { + return nil // Allow some time for new release to appear in Homebrew repo + } + log.Printf("\nNew release available: %s", color.GreenString(latest.Version.String())) + log.Printf("https://github.com/vespa-engine/vespa/releases/tag/v%s", latest.Version) + if usingHomebrew { + log.Printf("\nUpgrade by running:\n%s", color.CyanString("brew update && brew upgrade vespa-cli")) + } + return nil +} + +func latestRelease(cli *CLI) (release, error) { + req, err := http.NewRequest("GET", "https://api.github.com/repos/vespa-engine/vespa/releases", nil) + if err != nil { + return release{}, err + } + response, err := cli.httpClient.Do(req, time.Minute) + if err != nil { + return release{}, err + } + defer response.Body.Close() + + var ghReleases []githubRelease + dec := json.NewDecoder(response.Body) + if err := dec.Decode(&ghReleases); err != nil { + return release{}, err + } + if len(ghReleases) == 0 { + return release{}, nil // No releases found + } + + var releases []release + for _, r := range ghReleases { + v, err := version.Parse(r.TagName) + if err != nil { + return release{}, err + } + publishedAt, err := time.Parse(time.RFC3339, r.PublishedAt) + if err != nil { + return release{}, err + } + releases = append(releases, release{Version: v, PublishedAt: publishedAt}) + } + sort.Slice(releases, func(i, j int) bool { return releases[i].Version.Less(releases[j].Version) }) + return releases[len(releases)-1], nil +} + +func usingHomebrew(cli *CLI) bool { + selfPath, err := cli.exec.LookPath("vespa") + if err != nil { + return false + } + brewPrefix, err := cli.exec.Run("brew", "--prefix") + if err != nil { + return false + } + brewBin := filepath.Join(strings.TrimSpace(string(brewPrefix)), "bin") + string(os.PathSeparator) + return strings.HasPrefix(selfPath, brewBin) +} + +type githubRelease struct { + TagName string `json:"tag_name"` + PublishedAt string `json:"published_at"` +} + +type release struct { + Version version.Version + PublishedAt time.Time +} + +func (r release) isRecent() bool { + return time.Now().Before(r.PublishedAt.Add(time.Hour * 24)) +} diff --git a/client/go/internal/cli/cmd/version_test.go b/client/go/internal/cli/cmd/version_test.go new file mode 100644 index 00000000000..70eaf1814e7 --- /dev/null +++ b/client/go/internal/cli/cmd/version_test.go @@ -0,0 +1,45 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/mock" +) + +func TestVersion(t *testing.T) { + c := &mock.HTTPClient{} + c.NextResponseString(200, `[{"tag_name": "v1.2.3", "published_at": "2021-09-10T12:00:00Z"}]`) + + sp := &mock.Exec{} + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = c + cli.exec = sp + cli.isTerminal = func() bool { return true } + if err := cli.Run("version", "--color", "never"); err != nil { + t.Fatal(err) + } + assert.Equal(t, "", stderr.String()) + assert.Contains(t, stdout.String(), "Vespa CLI version 0.0.0-devel compiled with") + assert.Contains(t, stdout.String(), "New release available: 1.2.3\nhttps://github.com/vespa-engine/vespa/releases/tag/v1.2.3") +} + +func TestVersionCheckHomebrew(t *testing.T) { + c := &mock.HTTPClient{} + c.NextResponseString(200, `[{"tag_name": "v1.2.3", "published_at": "2021-09-10T12:00:00Z"}]`) + + sp := &mock.Exec{ProgramPath: "/usr/local/bin/vespa", CombinedOutput: "/usr/local"} + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = c + cli.exec = sp + cli.isTerminal = func() bool { return true } + if err := cli.Run("version", "--color", "never"); err != nil { + t.Fatal(err) + } + assert.Equal(t, "", stderr.String()) + assert.Contains(t, stdout.String(), "Vespa CLI version 0.0.0-devel compiled with") + assert.Contains(t, stdout.String(), "New release available: 1.2.3\n"+ + "https://github.com/vespa-engine/vespa/releases/tag/v1.2.3\n"+ + "\nUpgrade by running:\nbrew update && brew upgrade vespa-cli\n") +} diff --git a/client/go/internal/cli/cmd/vespa/main.go b/client/go/internal/cli/cmd/vespa/main.go new file mode 100644 index 00000000000..cca35ba8368 --- /dev/null +++ b/client/go/internal/cli/cmd/vespa/main.go @@ -0,0 +1,33 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Cobra commands main file +// Author: bratseth + +package main + +import ( + "fmt" + "os" + + "github.com/vespa-engine/vespa/client/go/internal/cli/cmd" +) + +func fatal(status int, err error) { + if err != nil { + fmt.Fprintln(os.Stderr, err) + } + os.Exit(status) +} + +func main() { + cli, err := cmd.New(os.Stdout, os.Stderr, os.Environ()) + if err != nil { + fatal(1, err) + } + if err := cli.Run(); err != nil { + if cliErr, ok := err.(cmd.ErrCLI); ok { + fatal(cliErr.Status, nil) + } else { + fatal(1, nil) + } + } +} diff --git a/client/go/internal/cli/config/config.go b/client/go/internal/cli/config/config.go new file mode 100644 index 00000000000..beffff6e257 --- /dev/null +++ b/client/go/internal/cli/config/config.go @@ -0,0 +1,94 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package config + +import ( + "fmt" + "io" + "os" + "path/filepath" + "sort" + "sync" + + "gopkg.in/yaml.v3" +) + +// Config represents a thread-safe key-value config, which can be marshalled to YAML. +type Config struct { + values map[string]string + mu sync.RWMutex +} + +// New creates a new config. +func New() *Config { return &Config{values: make(map[string]string)} } + +// Keys returns a sorted slice of keys set in this config. +func (c *Config) Keys() []string { + var keys []string + for k := range c.values { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// Get returns the value associated with key. +func (c *Config) Get(key string) (string, bool) { + c.mu.RLock() + defer c.mu.RUnlock() + v, ok := c.values[key] + return v, ok +} + +// Set associates key with value. +func (c *Config) Set(key, value string) { + c.mu.Lock() + defer c.mu.Unlock() + c.values[key] = value +} + +// Del removes the value associated with key. +func (c *Config) Del(key string) { + c.mu.Lock() + defer c.mu.Unlock() + delete(c.values, key) +} + +// Write writes config in YAML format to writer w. +func (c *Config) Write(w io.Writer) error { + c.mu.RLock() + defer c.mu.RUnlock() + if err := yaml.NewEncoder(w).Encode(c.values); err != nil { + return fmt.Errorf("failed to write config: %w", err) + } + return nil +} + +// WriteFile writes the config to a temporary file in the parent directory of filename, then renames the temporary file +// to filename. +func (c *Config) WriteFile(filename string) error { + dir := filepath.Dir(filename) + f, err := os.CreateTemp(dir, "config") + if err != nil { + return err + } + defer func() { + f.Close() + os.Remove(f.Name()) + }() + if err := c.Write(f); err != nil { + return err + } + if err := f.Close(); err != nil { + return err + } + return os.Rename(f.Name(), filename) +} + +// Read configuration in YAML format from reader r. +func Read(r io.Reader) (*Config, error) { + config := New() + if err := yaml.NewDecoder(r).Decode(config.values); err != nil { + return nil, err + } + return config, nil +} diff --git a/client/go/internal/cli/config/config_test.go b/client/go/internal/cli/config/config_test.go new file mode 100644 index 00000000000..1458771a5f5 --- /dev/null +++ b/client/go/internal/cli/config/config_test.go @@ -0,0 +1,42 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package config + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConfig(t *testing.T) { + config := New() + config.Set("key1", "value1") + config.Set("key2", "value2") + config.Set("key3", "value3") + assert.Equal(t, []string{"key1", "key2", "key3"}, config.Keys()) + + v, ok := config.Get("key3") + assert.True(t, ok) + assert.Equal(t, "value3", v) + + config.Del("key3") + _, ok = config.Get("key3") + assert.False(t, ok) + + var buf bytes.Buffer + require.Nil(t, config.Write(&buf)) + assert.Equal(t, "key1: value1\nkey2: value2\n", buf.String()) + + unmarshalled, err := Read(&buf) + require.Nil(t, err) + assert.Equal(t, config, unmarshalled) + + filename := filepath.Join(t.TempDir(), "config.yaml") + require.Nil(t, config.WriteFile(filename)) + data, err := os.ReadFile(filename) + require.Nil(t, err) + assert.Equal(t, "key1: value1\nkey2: value2\n", string(data)) +} |