diff options
Diffstat (limited to 'client/go/internal')
52 files changed, 900 insertions, 398 deletions
diff --git a/client/go/internal/admin/jvm/mem_options.go b/client/go/internal/admin/jvm/mem_options.go index d3b0d44c677..c78ee80dc80 100644 --- a/client/go/internal/admin/jvm/mem_options.go +++ b/client/go/internal/admin/jvm/mem_options.go @@ -55,8 +55,8 @@ func (opts *Options) MaybeAddHugepages(heapSize AmountOfMemory) { } func adjustAvailableMemory(measured AmountOfMemory) AmountOfMemory { - reserved := 1024 // MB - need_min := 64 // MB + reserved := 700 // MB -- keep in sync with com.yahoo.vespa.model.Host.memoryOverheadGb + need_min := 64 // MB available := measured.ToMB() if available > need_min+2*reserved { return MegaBytesOfMemory(available - reserved) diff --git a/client/go/internal/admin/jvm/mem_options_test.go b/client/go/internal/admin/jvm/mem_options_test.go index 3501e44c723..60cffc824e9 100644 --- a/client/go/internal/admin/jvm/mem_options_test.go +++ b/client/go/internal/admin/jvm/mem_options_test.go @@ -9,11 +9,11 @@ import ( func TestAdjustment(t *testing.T) { lastAdj := 64 - for i := 0; i < 4096; i++ { + for i := range 4096 { adj := adjustAvailableMemory(MegaBytesOfMemory(i)).ToMB() assert.True(t, int(adj) >= lastAdj) lastAdj = int(adj) } - adj := adjustAvailableMemory(MegaBytesOfMemory(31024)).ToMB() + adj := adjustAvailableMemory(MegaBytesOfMemory(30700)).ToMB() assert.Equal(t, 30000, int(adj)) } diff --git a/client/go/internal/admin/jvm/qr_start_cfg.go b/client/go/internal/admin/jvm/qr_start_cfg.go index a14ab0d1946..cbcc2c066ee 100644 --- a/client/go/internal/admin/jvm/qr_start_cfg.go +++ b/client/go/internal/admin/jvm/qr_start_cfg.go @@ -38,7 +38,7 @@ func (a *ApplicationContainer) getQrStartCfg() *QrStartConfig { var parsedJson QrStartConfig args := []string{ "-j", - "-w", "10", + "-w", "30", "-n", "search.config.qr-start", "-i", a.ConfigId(), } diff --git a/client/go/internal/admin/jvm/standalone_container.go b/client/go/internal/admin/jvm/standalone_container.go index 20031bc7725..fb615dcf4c2 100644 --- a/client/go/internal/admin/jvm/standalone_container.go +++ b/client/go/internal/admin/jvm/standalone_container.go @@ -37,7 +37,6 @@ func (a *StandaloneContainer) configureOptions() { opts := a.jvmOpts opts.ConfigureCpuCount(0) opts.AddCommonXX() - opts.AddOption("-XX:-OmitStackTraceInFastThrow") opts.AddCommonOpens() opts.AddCommonJdkProperties() a.addJdiscProperties() diff --git a/client/go/internal/admin/jvm/xx_options.go b/client/go/internal/admin/jvm/xx_options.go index 13b69e43dda..abc92f19bf2 100644 --- a/client/go/internal/admin/jvm/xx_options.go +++ b/client/go/internal/admin/jvm/xx_options.go @@ -19,5 +19,5 @@ func (opts *Options) AddCommonXX() { // not common after all: opts.AddOption("-XX:MaxJavaStackTraceDepth=1000000") // Aid debugging for slight cost in performance - opts.AddOption("-XX:+OmitStackTraceInFastThrow") + opts.AddOption("-XX:-OmitStackTraceInFastThrow") } diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/tail_unix.go b/client/go/internal/admin/vespa-wrapper/logfmt/tail_unix.go index ec2c53487be..180f7c84859 100644 --- a/client/go/internal/admin/vespa-wrapper/logfmt/tail_unix.go +++ b/client/go/internal/admin/vespa-wrapper/logfmt/tail_unix.go @@ -85,7 +85,7 @@ func (t *unixTail) openTail() { if err != nil { return } - for i := 0; i < n; i++ { + for i := range n { if t.lineBuf[i] == '\n' { sz, err = file.Seek(sz+int64(i+1), os.SEEK_SET) if err == nil { diff --git a/client/go/internal/cli/cmd/cert.go b/client/go/internal/cli/cmd/cert.go index 14e4861cec3..9e0b8f6805c 100644 --- a/client/go/internal/cli/cmd/cert.go +++ b/client/go/internal/cli/cmd/cert.go @@ -160,10 +160,10 @@ func doCertAdd(cli *CLI, overwriteCertificate bool, args []string) error { if pkg.HasCertificate() && !overwriteCertificate { return errHint(fmt.Errorf("application package '%s' already contains a certificate", pkg.Path), "Use -f flag to force overwriting") } - return maybeCopyCertificate(true, false, cli, target, pkg) + return requireCertificate(true, false, cli, target, pkg) } -func maybeCopyCertificate(force, ignoreZip bool, cli *CLI, target vespa.Target, pkg vespa.ApplicationPackage) error { +func requireCertificate(force, ignoreZip bool, cli *CLI, target vespa.Target, pkg vespa.ApplicationPackage) error { if pkg.IsZip() { if ignoreZip { cli.printWarning("Cannot verify existence of "+color.CyanString("security/clients.pem")+" since '"+pkg.Path+"' is compressed", @@ -175,10 +175,31 @@ func maybeCopyCertificate(force, ignoreZip bool, cli *CLI, target vespa.Target, return errHint(fmt.Errorf("cannot add certificate to compressed application package: '%s'", pkg.Path), hint) } } + tlsOptions, err := cli.config.readTLSOptions(target.Deployment().Application, target.Type()) + if err != nil { + return err + } if force { - return copyCertificate(cli, target, pkg) + return copyCertificate(tlsOptions, cli, pkg) } if pkg.HasCertificate() { + if cli.isCI() { + return nil // A matching certificate is not required in CI environments + } + if len(tlsOptions.CertificatePEM) == 0 { + return errHint(fmt.Errorf("no certificate exists for %s", target.Deployment().Application.String()), "Try (re)creating the certificate with 'vespa auth cert'") + } + matches, err := pkg.HasMatchingCertificate(tlsOptions.CertificatePEM) + if err != nil { + return err + } + if !matches { + return errHint(fmt.Errorf("certificate in %s does not match the stored key pair for %s", + filepath.Join("security", "clients.pem"), + target.Deployment().Application.String()), + "If this application was deployed using a different application ID in the past, the matching key pair may be stored under a different ID in "+cli.config.homeDir, + "Specify the matching application with --application, or add the current certificate to the package using --add-cert") + } return nil } if cli.isTerminal() { @@ -188,7 +209,7 @@ func maybeCopyCertificate(force, ignoreZip bool, cli *CLI, target vespa.Target, return err } if ok { - return copyCertificate(cli, target, pkg) + return copyCertificate(tlsOptions, cli, pkg) } } return errHint(fmt.Errorf("deployment to Vespa Cloud requires certificate in application package"), @@ -196,15 +217,7 @@ func maybeCopyCertificate(force, ignoreZip bool, cli *CLI, target vespa.Target, "Pass --add-cert to use the certificate of the current application") } -func copyCertificate(cli *CLI, target vespa.Target, pkg vespa.ApplicationPackage) error { - tlsOptions, err := cli.config.readTLSOptions(target.Deployment().Application, target.Type()) - if err != nil { - return err - } - hint := "Try generating the certificate with 'vespa auth cert'" - if tlsOptions.CertificateFile == "" { - return errHint(fmt.Errorf("no certificate exists for "+target.Deployment().Application.String()), hint) - } +func copyCertificate(tlsOptions vespa.TLSOptions, cli *CLI, pkg vespa.ApplicationPackage) error { data, err := os.ReadFile(tlsOptions.CertificateFile) if err != nil { return errHint(fmt.Errorf("could not read certificate file: %w", err)) diff --git a/client/go/internal/cli/cmd/clone.go b/client/go/internal/cli/cmd/clone.go index 8fd12eb9e6e..3df490079a6 100644 --- a/client/go/internal/cli/cmd/clone.go +++ b/client/go/internal/cli/cmd/clone.go @@ -32,8 +32,7 @@ func newCloneCmd(cli *CLI) *cobra.Command { cmd := &cobra.Command{ Use: "clone sample-application-path target-directory", Short: "Create files and directory structure from a Vespa sample application", - Long: `Create files and directory structure from a Vespa sample application -from a sample application. + Long: `Create files and directory structure from a Vespa sample application. Sample applications are downloaded from https://github.com/vespa-engine/sample-apps. diff --git a/client/go/internal/cli/cmd/config.go b/client/go/internal/cli/cmd/config.go index fc6d69aab61..dfe48118340 100644 --- a/client/go/internal/cli/cmd/config.go +++ b/client/go/internal/cli/cmd/config.go @@ -35,7 +35,7 @@ func newConfigCmd() *cobra.Command { This command allows setting persistent values for the global flags found in Vespa CLI. On future invocations the flag can then be omitted as it is read - from the config file instead. +from the config file instead. Configuration is written to $HOME/.vespa by default. This path can be overridden by setting the VESPA_CLI_HOME environment variable. @@ -114,8 +114,7 @@ zone Specifies a custom zone to use when connecting to a Vespa Cloud application. This is only relevant for cloud and hosted targets and defaults to a dev zone. See https://cloud.vespa.ai/en/reference/zones for available zones. Examples: -dev.aws-us-east-1c, dev.gcp-us-central1-f, perf.aws-us-east-1c -`, +dev.aws-us-east-1c, dev.gcp-us-central1-f, perf.aws-us-east-1c`, DisableAutoGenTag: true, SilenceUsage: false, Args: cobra.MinimumNArgs(1), @@ -143,8 +142,7 @@ $ vespa config set application my-tenant.my-application.my-instance $ vespa config set instance other-instance # Set an option in local configuration, for the current application only -$ vespa config set --local wait 600 -`, +$ vespa config set --local zone perf.us-north-1`, DisableAutoGenTag: true, SilenceUsage: true, Args: cobra.ExactArgs(2), @@ -180,8 +178,7 @@ Unsetting a configuration option will reset it to its default value, which may b $ vespa config unset target # Stop overriding application option in local config -$ vespa config unset --local application -`, +$ vespa config unset --local application`, DisableAutoGenTag: true, SilenceUsage: true, Args: cobra.ExactArgs(1), @@ -216,8 +213,7 @@ application, i.e. it takes into account any local configuration located in `, Example: `$ vespa config get $ vespa config get target -$ vespa config get --local -`, +$ vespa config get --local`, Args: cobra.MaximumNArgs(1), DisableAutoGenTag: true, SilenceUsage: true, @@ -429,51 +425,76 @@ func (c *Config) privateKeyPath(app vespa.ApplicationID, targetType string) (cre return c.credentialsFile(app, targetType, false) } -func (c *Config) readTLSOptions(app vespa.ApplicationID, targetType string) (vespa.TLSOptions, error) { - _, trustAll := c.environment["VESPA_CLI_DATA_PLANE_TRUST_ALL"] - cert, certOk := c.environment["VESPA_CLI_DATA_PLANE_CERT"] - key, keyOk := c.environment["VESPA_CLI_DATA_PLANE_KEY"] - caCertText, caCertOk := c.environment["VESPA_CLI_DATA_PLANE_CA_CERT"] - options := vespa.TLSOptions{TrustAll: trustAll} - // CA certificate +func (c *Config) caCertificatePEM() ([]byte, string, error) { + envVar := "VESPA_CLI_DATA_PLANE_CA_CERT" + caCertText, caCertOk := c.environment[envVar] if caCertOk { - options.CACertificate = []byte(caCertText) - } else if caCertFile := c.caCertificatePath(); caCertFile != "" { + return []byte(caCertText), envVar, nil + } + if caCertFile := c.caCertificatePath(); caCertFile != "" { b, err := os.ReadFile(caCertFile) if err != nil { - return options, err + return nil, "", err } - options.CACertificate = b - options.CACertificateFile = caCertFile + return b, caCertFile, nil } - // Certificate and private key - if certOk && keyOk { - kp, err := tls.X509KeyPair([]byte(cert), []byte(key)) + return nil, "", nil +} + +func (c *Config) credentialsPEM(envVar string, credentialsFile credentialsFile) ([]byte, string, error) { + if pem, ok := c.environment[envVar]; ok { + return []byte(pem), envVar, nil + } + pem, err := os.ReadFile(credentialsFile.path) + if err != nil { + if os.IsNotExist(err) && credentialsFile.optional { + return nil, "", nil + } + return nil, "", err + } + return []byte(pem), credentialsFile.path, nil +} + +func (c *Config) readTLSOptions(app vespa.ApplicationID, targetType string) (vespa.TLSOptions, error) { + var options vespa.TLSOptions + // Certificate + if certPath, err := c.certificatePath(app, targetType); err == nil { + certPEM, certFile, err := c.credentialsPEM("VESPA_CLI_DATA_PLANE_CERT", certPath) if err != nil { return vespa.TLSOptions{}, err } - options.KeyPair = []tls.Certificate{kp} + options.CertificatePEM = certPEM + options.CertificateFile = certFile } else { - keyFile, err := c.privateKeyPath(app, targetType) + return vespa.TLSOptions{}, err + } + // Private key + if keyPath, err := c.privateKeyPath(app, targetType); err == nil { + keyPEM, keyFile, err := c.credentialsPEM("VESPA_CLI_DATA_PLANE_KEY", keyPath) if err != nil { return vespa.TLSOptions{}, err } - certFile, err := c.certificatePath(app, targetType) + options.PrivateKeyPEM = keyPEM + options.PrivateKeyFile = keyFile + } else { + return vespa.TLSOptions{}, err + + } + // CA certificate + _, options.TrustAll = c.environment["VESPA_CLI_DATA_PLANE_TRUST_ALL"] + caCertificate, caCertificateFile, err := c.caCertificatePEM() + if err != nil { + return vespa.TLSOptions{}, err + } + options.CACertificatePEM = caCertificate + options.CACertificateFile = caCertificateFile + // Key pair + if len(options.CertificatePEM) > 0 && len(options.PrivateKeyPEM) > 0 { + kp, err := tls.X509KeyPair(options.CertificatePEM, options.PrivateKeyPEM) if err != nil { return vespa.TLSOptions{}, err } - kp, err := tls.LoadX509KeyPair(certFile.path, keyFile.path) - allowMissing := os.IsNotExist(err) && keyFile.optional && certFile.optional - if err == nil { - options.KeyPair = []tls.Certificate{kp} - options.PrivateKeyFile = keyFile.path - options.CertificateFile = certFile.path - } else if err != nil && !allowMissing { - return vespa.TLSOptions{}, err - } - } - // If we found a key pair, parse it and check expiry - if options.KeyPair != nil { + options.KeyPair = []tls.Certificate{kp} cert, err := x509.ParseCertificate(options.KeyPair[0].Certificate[0]) if err != nil { return vespa.TLSOptions{}, err @@ -513,7 +534,7 @@ func (c *Config) authConfigPath() string { return filepath.Join(c.homeDir, "auth.json") } -func (c *Config) readAPIKey(cli *CLI, system vespa.System, tenantName string) ([]byte, error) { +func (c *Config) readAPIKey(cli *CLI, tenantName string) ([]byte, error) { if override, ok := c.apiKeyFromEnv(); ok { return override, nil } diff --git a/client/go/internal/cli/cmd/config_test.go b/client/go/internal/cli/cmd/config_test.go index b13f8365f5f..6c9321b3219 100644 --- a/client/go/internal/cli/cmd/config_test.go +++ b/client/go/internal/cli/cmd/config_test.go @@ -156,19 +156,19 @@ func assertConfigCommandErr(t *testing.T, configHome, expected string, args ...s func TestReadAPIKey(t *testing.T) { cli, _, _ := newTestCLI(t) - key, err := cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + key, err := cli.config.readAPIKey(cli, "t1") assert.Nil(t, key) require.NotNil(t, err) // From default path when it exists require.Nil(t, os.WriteFile(filepath.Join(cli.config.homeDir, "t1.api-key.pem"), []byte("foo"), 0600)) - key, err = cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + key, err = cli.config.readAPIKey(cli, "t1") require.Nil(t, err) assert.Equal(t, []byte("foo"), key) // Cloud CI never reads key from disk as it's not expected to have any cli, _, _ = newTestCLI(t, "VESPA_CLI_CLOUD_CI=true") - key, err = cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + key, err = cli.config.readAPIKey(cli, "t1") require.Nil(t, err) assert.Nil(t, key) @@ -176,20 +176,20 @@ func TestReadAPIKey(t *testing.T) { keyFile := filepath.Join(t.TempDir(), "key") require.Nil(t, os.WriteFile(keyFile, []byte("bar"), 0600)) cli, _, _ = newTestCLI(t, "VESPA_CLI_API_KEY_FILE="+keyFile) - key, err = cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + key, err = cli.config.readAPIKey(cli, "t1") require.Nil(t, err) assert.Equal(t, []byte("bar"), key) // From key specified in environment cli, _, _ = newTestCLI(t, "VESPA_CLI_API_KEY=baz") - key, err = cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + key, err = cli.config.readAPIKey(cli, "t1") require.Nil(t, err) assert.Equal(t, []byte("baz"), key) // Prefer Auth0 if we have auth config cli, _, _ = newTestCLI(t) require.Nil(t, os.WriteFile(filepath.Join(cli.config.homeDir, "auth.json"), []byte("foo"), 0600)) - key, err = cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + key, err = cli.config.readAPIKey(cli, "t1") require.Nil(t, err) assert.Nil(t, key) } @@ -209,9 +209,14 @@ func TestConfigReadTLSOptions(t *testing.T) { assertTLSOptions(t, homeDir, app, vespa.TargetLocal, vespa.TLSOptions{ - TrustAll: true, - CACertificate: []byte("cacert"), - KeyPair: []tls.Certificate{keyPair}, + TrustAll: true, + KeyPair: []tls.Certificate{keyPair}, + CACertificatePEM: []byte("cacert"), + CertificatePEM: pemCert, + PrivateKeyPEM: pemKey, + CACertificateFile: "VESPA_CLI_DATA_PLANE_CA_CERT", + CertificateFile: "VESPA_CLI_DATA_PLANE_CERT", + PrivateKeyFile: "VESPA_CLI_DATA_PLANE_KEY", }, "VESPA_CLI_DATA_PLANE_TRUST_ALL=true", "VESPA_CLI_DATA_PLANE_CA_CERT=cacert", @@ -230,7 +235,9 @@ func TestConfigReadTLSOptions(t *testing.T) { vespa.TargetLocal, vespa.TLSOptions{ KeyPair: []tls.Certificate{keyPair}, - CACertificate: []byte("cacert"), + CACertificatePEM: []byte("cacert"), + CertificatePEM: pemCert, + PrivateKeyPEM: pemKey, CACertificateFile: caCertFile, CertificateFile: certFile, PrivateKeyFile: keyFile, @@ -249,6 +256,8 @@ func TestConfigReadTLSOptions(t *testing.T) { vespa.TargetLocal, vespa.TLSOptions{ KeyPair: []tls.Certificate{keyPair}, + CertificatePEM: pemCert, + PrivateKeyPEM: pemKey, CertificateFile: defaultCertFile, PrivateKeyFile: defaultKeyFile, }, diff --git a/client/go/internal/cli/cmd/curl.go b/client/go/internal/cli/cmd/curl.go index c7297574a32..63f6ef8c592 100644 --- a/client/go/internal/cli/cmd/curl.go +++ b/client/go/internal/cli/cmd/curl.go @@ -37,7 +37,7 @@ $ vespa curl -- -v --data-urlencode "yql=select * from music where album contain if err != nil { return err } - waiter := cli.waiter(time.Duration(waitSecs) * time.Second) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) service, err := waiter.Service(target, cli.config.cluster()) if err != nil { return err diff --git a/client/go/internal/cli/cmd/deploy.go b/client/go/internal/cli/cmd/deploy.go index 9ae6676bc17..fac779c8241 100644 --- a/client/go/internal/cli/cmd/deploy.go +++ b/client/go/internal/cli/cmd/deploy.go @@ -5,6 +5,7 @@ package cmd import ( + "errors" "fmt" "io" "log" @@ -29,19 +30,22 @@ func newDeployCmd(cli *CLI) *cobra.Command { Short: "Deploy (prepare and activate) an application package", Long: `Deploy (prepare and activate) an application package. -When this returns successfully the application package has been validated -and activated on config servers. The process of applying it on individual nodes -has started but may not have completed. +An application package defines a deployable Vespa application. See +https://docs.vespa.ai/en/reference/application-packages-reference.html for +details about the files contained in this package. -If application directory is not specified, it defaults to working directory. +To get started, 'vespa clone' can be used to download a sample application. + +This command deploys an application package. When deploy returns successfully +the application package has been validated and activated on config servers. The +process of applying it on individual nodes has started but may not have +completed. -When deploying to Vespa Cloud the system can be overridden by setting the -environment variable VESPA_CLI_CLOUD_SYSTEM. This is intended for internal use -only. +If application directory is not specified, it defaults to working directory. -In Vespa Cloud you may override the Vespa runtime version for your deployment. -This option should only be used if you have a reason for using a specific -version. By default Vespa Cloud chooses a suitable version for you. +In Vespa Cloud you may override the Vespa runtime version (--version) for your +deployment. This option should only be used if you have a reason for using a +specific version. By default Vespa Cloud chooses a suitable version for you. `, Example: `$ vespa deploy . $ vespa deploy -t cloud @@ -59,7 +63,6 @@ $ vespa deploy -t cloud -z perf.aws-us-east-1c`, if err != nil { return err } - timeout := time.Duration(waitSecs) * time.Second opts := vespa.DeploymentOptions{ApplicationPackage: pkg, Target: target} if versionArg != "" { version, err := version.Parse(versionArg) @@ -69,19 +72,26 @@ $ vespa deploy -t cloud -z perf.aws-us-east-1c`, opts.Version = version } if target.Type() == vespa.TargetCloud { - if err := maybeCopyCertificate(copyCert, true, cli, target, pkg); err != nil { + if err := requireCertificate(copyCert, true, cli, target, pkg); err != nil { return err } } - waiter := cli.waiter(timeout) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) if _, err := waiter.DeployService(target); err != nil { return err } var result vespa.PrepareResult - if err := cli.spinner(cli.Stderr, "Uploading application package...", func() error { + err = cli.spinner(cli.Stderr, "Uploading application package...", func() error { result, err = vespa.Deploy(opts) return err - }); err != nil { + }) + if err != nil { + if target.IsCloud() && errors.Is(err, vespa.ErrUnauthorized) { + return errHint(err, + "You do not have access to the tenant "+color.CyanString(target.Deployment().Application.Tenant), + "You may need to create the tenant at "+color.CyanString(target.Deployment().System.ConsoleURL+"/tenant"), + "If the tenant already exists you may need to run 'vespa auth login' to gain access to it") + } return err } log.Println() @@ -95,7 +105,7 @@ $ vespa deploy -t cloud -z perf.aws-us-east-1c`, log.Printf("\nUse %s for deployment status, or follow this deployment at", color.CyanString("vespa status deployment")) log.Print(color.CyanString(opts.Target.Deployment().System.ConsoleRunURL(opts.Target.Deployment(), result.ID))) } - return waitForDeploymentReady(cli, target, result.ID, timeout) + return waitForVespaReady(target, result.ID, waiter) }, } cmd.Flags().StringVarP(&logLevelArg, "log-level", "l", "error", `Log level for Vespa logs. Must be "error", "warning", "info" or "debug"`) @@ -157,8 +167,7 @@ func newActivateCmd(cli *CLI) *cobra.Command { if err != nil { return err } - timeout := time.Duration(waitSecs) * time.Second - waiter := cli.waiter(timeout) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) if _, err := waiter.DeployService(target); err != nil { return err } @@ -168,23 +177,30 @@ func newActivateCmd(cli *CLI) *cobra.Command { return err } cli.printSuccess("Activated application with session ", sessionID) - return waitForDeploymentReady(cli, target, sessionID, timeout) + return waitForVespaReady(target, sessionID, waiter) }, } cli.bindWaitFlag(cmd, 0, &waitSecs) return cmd } -func waitForDeploymentReady(cli *CLI, target vespa.Target, sessionOrRunID int64, timeout time.Duration) error { - if timeout == 0 { - return nil - } - waiter := cli.waiter(timeout) - if _, err := waiter.Deployment(target, sessionOrRunID); err != nil { - return err +func waitForVespaReady(target vespa.Target, sessionOrRunID int64, waiter *Waiter) error { + fastWait := waiter.FastWaitOn(target) + hasTimeout := waiter.Timeout > 0 + if fastWait || hasTimeout { + // Wait for deployment convergence + if _, err := waiter.Deployment(target, sessionOrRunID); err != nil { + return err + } + // Wait for healthy services where we expect them to be reachable (cloud and local). When using a custom target, + // we do not wait for services as there is no guarantee that they are reachable from the machine executing + // deploy. + if hasTimeout && (target.IsCloud() || target.Type() == vespa.TargetLocal) { + _, err := waiter.Services(target) + return err + } } - _, err := waiter.Services(target) - return err + return nil } func printPrepareLog(stderr io.Writer, result vespa.PrepareResult) { diff --git a/client/go/internal/cli/cmd/deploy_test.go b/client/go/internal/cli/cmd/deploy_test.go index 4e32b9bbd60..3f71a59e682 100644 --- a/client/go/internal/cli/cmd/deploy_test.go +++ b/client/go/internal/cli/cmd/deploy_test.go @@ -5,8 +5,10 @@ package cmd import ( + "archive/zip" "bytes" "io" + "os" "path/filepath" "strconv" "strings" @@ -22,7 +24,7 @@ func TestDeployCloud(t *testing.T) { pkgDir := filepath.Join(t.TempDir(), "app") createApplication(t, pkgDir, false, false) - cli, stdout, stderr := newTestCLI(t, "CI=true", "NO_COLOR=true") + cli, stdout, stderr := newTestCLI(t, "NO_COLOR=true") httpClient := &mock.HTTPClient{} httpClient.NextResponseString(200, `ok`) cli.httpClient = httpClient @@ -35,11 +37,12 @@ func TestDeployCloud(t *testing.T) { stderr.Reset() require.NotNil(t, cli.Run("deploy", pkgDir)) + apiKeyWarning := "Warning: Authenticating with API key, intended for use in CI environments.\nHint: Authenticate with 'vespa auth login' instead\n" certError := `Error: deployment to Vespa Cloud requires certificate in application package Hint: See https://cloud.vespa.ai/en/security/guide Hint: Pass --add-cert to use the certificate of the current application ` - assert.Equal(t, certError, stderr.String()) + assert.Equal(t, apiKeyWarning+certError, stderr.String()) require.Nil(t, cli.Run("deploy", "--add-cert", "--wait=0", pkgDir)) assert.Contains(t, stdout.String(), "Success: Triggered deployment") @@ -55,11 +58,89 @@ Hint: Pass --add-cert to use the certificate of the current application buf.WriteString("wat\nthe\nfck\nn\n") cli.Stdin = &buf require.NotNil(t, cli.Run("deploy", "--add-cert=false", "--wait=0", pkgDir2)) - warning := "Warning: Application package does not contain security/clients.pem, which is required for deployments to Vespa Cloud\n" + warning := apiKeyWarning + "Warning: Application package does not contain security/clients.pem, which is required for deployments to Vespa Cloud\n" assert.Equal(t, warning+strings.Repeat("Error: please answer 'y' or 'n'\n", 3)+certError, stderr.String()) buf.WriteString("y\n") require.Nil(t, cli.Run("deploy", "--add-cert=false", "--wait=0", pkgDir2)) assert.Contains(t, stdout.String(), "Success: Triggered deployment") + + // Missing application certificate is detected + stderr.Reset() + require.NotNil(t, cli.Run("deploy", "--application=t1.a2.i2", pkgDir2)) + assert.Equal(t, apiKeyWarning+"Error: no certificate exists for t1.a2.i2\nHint: Try (re)creating the certificate with 'vespa auth cert'\n", stderr.String()) + + // Mismatching certificate is detected + stdout.Reset() + stderr.Reset() + assert.Nil(t, cli.Run("auth", "cert", "--application=t1.a1.i1", "-f", "--no-add")) + require.NotNil(t, cli.Run("deploy", "--application=t1.a1.i1", pkgDir2)) + assert.Equal(t, apiKeyWarning+`Error: certificate in security/clients.pem does not match the stored key pair for t1.a1.i1 +Hint: If this application was deployed using a different application ID in the past, the matching key pair may be stored under a different ID in `+ + cli.config.homeDir+"\nHint: Specify the matching application with --application, or add the current certificate to the package using --add-cert\n", + stderr.String()) +} + +func TestDeployCloudFastWait(t *testing.T) { + pkgDir := filepath.Join(t.TempDir(), "app") + createApplication(t, pkgDir, false, false) + + cli, stdout, stderr := newTestCLI(t, "CI=true") + httpClient := &mock.HTTPClient{} + cli.httpClient = httpClient + + app := vespa.ApplicationID{Tenant: "t1", Application: "a1", Instance: "i1"} + assert.Nil(t, cli.Run("config", "set", "application", app.String())) + assert.Nil(t, cli.Run("config", "set", "target", "cloud")) + assert.Nil(t, cli.Run("auth", "api-key")) + assert.Nil(t, cli.Run("auth", "cert", pkgDir)) + + // Deployment completes quickly + httpClient.NextResponseString(200, `ok`) + httpClient.NextResponseString(200, `{"active": false, "status": "success"}`) + require.Nil(t, cli.Run("deploy", pkgDir)) + assert.Contains(t, stdout.String(), "Success: Triggered deployment") + assert.True(t, httpClient.Consumed()) + + // Deployment fails quickly + stdout.Reset() + stderr.Reset() + httpClient.NextResponseString(200, `ok`) + httpClient.NextResponseString(200, `{"active": false, "status": "unsuccesful"}`) + require.NotNil(t, cli.Run("deploy", pkgDir)) + assert.Equal(t, stderr.String(), "Error: deployment run 0 not yet complete after waiting up to 2s: aborting wait: deployment failed: run 0 ended with unsuccessful status: unsuccesful\n") + assert.True(t, httpClient.Consumed()) + + // Deployment which is running does not return error + stdout.Reset() + stderr.Reset() + httpClient.NextResponseString(200, `ok`) + httpClient.NextResponseString(200, `{"active": true, "status": "running"}`) + require.Nil(t, cli.Run("deploy", pkgDir)) + assert.Contains(t, stdout.String(), "Success: Triggered deployment") + assert.True(t, httpClient.Consumed()) +} + +func TestDeployCloudUnauthorized(t *testing.T) { + pkgDir := filepath.Join(t.TempDir(), "app") + createApplication(t, pkgDir, false, false) + + cli, _, stderr := newTestCLI(t, "CI=true") + httpClient := &mock.HTTPClient{} + cli.httpClient = httpClient + + app := vespa.ApplicationID{Tenant: "t1", Application: "a1", Instance: "i1"} + assert.Nil(t, cli.Run("config", "set", "application", app.String())) + assert.Nil(t, cli.Run("config", "set", "target", "cloud")) + assert.Nil(t, cli.Run("auth", "api-key")) + assert.Nil(t, cli.Run("auth", "cert", pkgDir)) + httpClient.NextResponseString(403, "bugger off") + require.NotNil(t, cli.Run("deploy", pkgDir)) + assert.Equal(t, `Error: deployment failed: unauthorized (status 403) +bugger off +Hint: You do not have access to the tenant t1 +Hint: You may need to create the tenant at https://console.vespa-cloud.com/tenant +Hint: If the tenant already exists you may need to run 'vespa auth login' to gain access to it +`, stderr.String()) } func TestDeployWait(t *testing.T) { @@ -94,8 +175,7 @@ func TestPrepareZip(t *testing.T) { } func TestActivateZip(t *testing.T) { - assertActivate("testdata/applications/withTarget/target/application.zip", - []string{"activate", "--wait=0", "testdata/applications/withTarget/target/application.zip"}, t) + assertActivate([]string{"activate", "--wait=0", "testdata/applications/withTarget/target/application.zip"}, t) } func TestDeployZip(t *testing.T) { @@ -145,14 +225,40 @@ func TestDeployApplicationDirectoryWithPomAndEmptyTarget(t *testing.T) { stderr.String()) } +func TestDeployIncludesExpectedFiles(t *testing.T) { + cli, stdout, _ := newTestCLI(t) + client := &mock.HTTPClient{} + cli.httpClient = client + assert.Nil(t, cli.Run("deploy", "--wait=0", "testdata/applications/withSource")) + applicationPackage := "testdata/applications/withSource/src/main/application" + assert.Equal(t, + "\nSuccess: Deployed '"+applicationPackage+"' with session ID 0\n", + stdout.String()) + + zipName := filepath.Join(t.TempDir(), "tmp.zip") + f, err := os.Create(zipName) + assert.Nil(t, err) + if _, err := io.Copy(f, client.LastRequest.Body); err != nil { + t.Fatal(err) + } + zr, err := zip.OpenReader(zipName) + assert.Nil(t, err) + defer zr.Close() + var zipFiles []string + for _, f := range zr.File { + zipFiles = append(zipFiles, f.Name) + } + assert.Equal(t, []string{".vespaignore", "hosts.xml", "schemas/msmarco.sd", "services.xml"}, zipFiles) +} + func TestDeployApplicationPackageErrorWithUnexpectedNonJson(t *testing.T) { - assertApplicationPackageError(t, "deploy", 401, + assertApplicationPackageError(t, "deploy", 400, "Raw text error", "Raw text error") } func TestDeployApplicationPackageErrorWithUnexpectedJson(t *testing.T) { - assertApplicationPackageError(t, "deploy", 401, + assertApplicationPackageError(t, "deploy", 400, `{ "some-unexpected-json": "Invalid XML, error in services.xml: element \"nosuch\" not allowed here" }`, @@ -211,7 +317,7 @@ func assertPrepare(applicationPackage string, arguments []string, t *testing.T) assert.Equal(t, "PUT", client.Requests[1].Method) } -func assertActivate(applicationPackage string, arguments []string, t *testing.T) { +func assertActivate(arguments []string, t *testing.T) { t.Helper() client := &mock.HTTPClient{} cli, stdout, _ := newTestCLI(t) @@ -264,7 +370,7 @@ func assertApplicationPackageError(t *testing.T, cmd string, status int, expecte args = append(args, "testdata/applications/withTarget/target/application.zip") assert.NotNil(t, cli.Run(args...)) assert.Equal(t, - "Error: invalid application package (Status "+strconv.Itoa(status)+")\n"+expectedMessage+"\n", + "Error: invalid application package (status "+strconv.Itoa(status)+")\n"+expectedMessage+"\n", stderr.String()) } @@ -276,6 +382,6 @@ func assertDeployServerError(t *testing.T, status int, errorMessage string) { cli.httpClient = client assert.NotNil(t, cli.Run("deploy", "--wait=0", "testdata/applications/withTarget/target/application.zip")) assert.Equal(t, - "Error: error from deploy API at 127.0.0.1:19071 (Status "+strconv.Itoa(status)+"):\n"+errorMessage+"\n", + "Error: error from deploy API at 127.0.0.1:19071 (status "+strconv.Itoa(status)+"):\n"+errorMessage+"\n", stderr.String()) } diff --git a/client/go/internal/cli/cmd/destroy.go b/client/go/internal/cli/cmd/destroy.go index a7beff2e4b4..51040c0af8b 100644 --- a/client/go/internal/cli/cmd/destroy.go +++ b/client/go/internal/cli/cmd/destroy.go @@ -23,14 +23,13 @@ When run interactively, the command will prompt for confirmation before removing the application. When run non-interactively, the command will refuse to remove the application unless the --force option is given. -This command can only be used to remove non-production deployments. See -https://cloud.vespa.ai/en/deleting-applications for how to remove -production deployments. This command can only be used for deployments to -Vespa Cloud, for other systems destroy an application by cleaning up -containers in use by the application, see e.g -https://github.com/vespa-engine/sample-apps/tree/master/examples/operations/multinode-HA#clean-up-after-testing +This command can only be used to remove non-production deployments, in Vespa +Cloud. See https://cloud.vespa.ai/en/deleting-applications for how to remove +production deployments. -`, +For other systems, destroy the application by removing the +containers in use by the application. For example: +https://github.com/vespa-engine/sample-apps/tree/master/examples/operations/multinode-HA#clean-up-after-testing`, Example: `$ vespa destroy $ vespa destroy -a mytenant.myapp.myinstance $ vespa destroy --force`, diff --git a/client/go/internal/cli/cmd/document.go b/client/go/internal/cli/cmd/document.go index 0393a9b2595..c9f0c780be3 100644 --- a/client/go/internal/cli/cmd/document.go +++ b/client/go/internal/cli/cmd/document.go @@ -27,8 +27,8 @@ func addDocumentFlags(cli *CLI, cmd *cobra.Command, printCurl *bool, timeoutSecs cli.bindWaitFlag(cmd, 0, waitSecs) } -func documentClient(cli *CLI, timeoutSecs, waitSecs int, printCurl bool) (*document.Client, *vespa.Service, error) { - docService, err := documentService(cli, waitSecs) +func documentClient(cli *CLI, timeoutSecs int, waiter *Waiter, printCurl bool) (*document.Client, *vespa.Service, error) { + docService, err := documentService(cli, waiter) if err != nil { return nil, nil, err } @@ -47,8 +47,8 @@ func documentClient(cli *CLI, timeoutSecs, waitSecs int, printCurl bool) (*docum return client, docService, nil } -func sendOperation(op document.Operation, args []string, timeoutSecs, waitSecs int, printCurl bool, cli *CLI) error { - client, service, err := documentClient(cli, timeoutSecs, waitSecs, printCurl) +func sendOperation(op document.Operation, args []string, timeoutSecs int, waiter *Waiter, printCurl bool, cli *CLI) error { + client, service, err := documentClient(cli, timeoutSecs, waiter, printCurl) if err != nil { return err } @@ -91,8 +91,8 @@ func sendOperation(op document.Operation, args []string, timeoutSecs, waitSecs i return printResult(cli, operationResult(false, doc, service, result), false) } -func readDocument(id string, timeoutSecs, waitSecs int, printCurl bool, cli *CLI) error { - client, service, err := documentClient(cli, timeoutSecs, waitSecs, printCurl) +func readDocument(id string, timeoutSecs int, waiter *Waiter, printCurl bool, cli *CLI, fieldSet string) error { + client, service, err := documentClient(cli, timeoutSecs, waiter, printCurl) if err != nil { return err } @@ -100,7 +100,7 @@ func readDocument(id string, timeoutSecs, waitSecs int, printCurl bool, cli *CLI if err != nil { return err } - result := client.Get(docId) + result := client.Get(docId, fieldSet) return printResult(cli, operationResult(true, document.Document{Id: docId}, service, result), true) } @@ -146,7 +146,8 @@ should be used instead of this.`, SilenceUsage: true, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return sendOperation(-1, args, timeoutSecs, waitSecs, printCurl, cli) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) + return sendOperation(-1, args, timeoutSecs, waiter, printCurl, cli) }, } addDocumentFlags(cli, cmd, &printCurl, &timeoutSecs, &waitSecs) @@ -171,7 +172,8 @@ $ vespa document put id:mynamespace:music::a-head-full-of-dreams src/test/resour DisableAutoGenTag: true, SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { - return sendOperation(document.OperationPut, args, timeoutSecs, waitSecs, printCurl, cli) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) + return sendOperation(document.OperationPut, args, timeoutSecs, waiter, printCurl, cli) }, } addDocumentFlags(cli, cmd, &printCurl, &timeoutSecs, &waitSecs) @@ -195,7 +197,8 @@ $ vespa document update id:mynamespace:music::a-head-full-of-dreams src/test/res DisableAutoGenTag: true, SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { - return sendOperation(document.OperationUpdate, args, timeoutSecs, waitSecs, printCurl, cli) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) + return sendOperation(document.OperationUpdate, args, timeoutSecs, waiter, printCurl, cli) }, } addDocumentFlags(cli, cmd, &printCurl, &timeoutSecs, &waitSecs) @@ -219,8 +222,9 @@ $ vespa document remove id:mynamespace:music::a-head-full-of-dreams`, DisableAutoGenTag: true, SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) if strings.HasPrefix(args[0], "id:") { - client, service, err := documentClient(cli, timeoutSecs, waitSecs, printCurl) + client, service, err := documentClient(cli, timeoutSecs, waiter, printCurl) if err != nil { return err } @@ -232,7 +236,7 @@ $ vespa document remove id:mynamespace:music::a-head-full-of-dreams`, result := client.Send(doc) return printResult(cli, operationResult(false, doc, service, result), false) } else { - return sendOperation(document.OperationRemove, args, timeoutSecs, waitSecs, printCurl, cli) + return sendOperation(document.OperationRemove, args, timeoutSecs, waiter, printCurl, cli) } }, } @@ -245,6 +249,7 @@ func newDocumentGetCmd(cli *CLI) *cobra.Command { printCurl bool timeoutSecs int waitSecs int + fieldSet string ) cmd := &cobra.Command{ Use: "get id", @@ -254,19 +259,20 @@ func newDocumentGetCmd(cli *CLI) *cobra.Command { SilenceUsage: true, Example: `$ vespa document get id:mynamespace:music::a-head-full-of-dreams`, RunE: func(cmd *cobra.Command, args []string) error { - return readDocument(args[0], timeoutSecs, waitSecs, printCurl, cli) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) + return readDocument(args[0], timeoutSecs, waiter, printCurl, cli, fieldSet) }, } + cmd.Flags().StringVar(&fieldSet, "field-set", "", "Fields to include when reading document") addDocumentFlags(cli, cmd, &printCurl, &timeoutSecs, &waitSecs) return cmd } -func documentService(cli *CLI, waitSecs int) (*vespa.Service, error) { +func documentService(cli *CLI, waiter *Waiter) (*vespa.Service, error) { target, err := cli.target(targetOptions{}) if err != nil { return nil, err } - waiter := cli.waiter(time.Duration(waitSecs) * time.Second) return waiter.Service(target, cli.config.cluster()) } diff --git a/client/go/internal/cli/cmd/feed.go b/client/go/internal/cli/cmd/feed.go index 69b847547a9..18906d1a66e 100644 --- a/client/go/internal/cli/cmd/feed.go +++ b/client/go/internal/cli/cmd/feed.go @@ -18,8 +18,9 @@ import ( func addFeedFlags(cli *CLI, cmd *cobra.Command, options *feedOptions) { cmd.PersistentFlags().IntVar(&options.connections, "connections", 8, "The number of connections to use") - cmd.PersistentFlags().StringVar(&options.compression, "compression", "auto", `Compression mode to use. Default is "auto" which compresses large documents. Must be "auto", "gzip" or "none"`) + cmd.PersistentFlags().StringVar(&options.compression, "compression", "auto", `Whether to compress the document data when sending the HTTP request. Default is "auto", which compresses large documents. Must be "auto", "gzip" or "none"`) cmd.PersistentFlags().IntVar(&options.timeoutSecs, "timeout", 0, "Individual feed operation timeout in seconds. 0 to disable (default 0)") + cmd.Flags().StringSliceVarP(&options.headers, "header", "", nil, "Add a header to all HTTP requests, on the format 'Header: Value'. This can be specified multiple times") cmd.PersistentFlags().IntVar(&options.doomSecs, "deadline", 0, "Exit if this number of seconds elapse without any successful operations. 0 to disable (default 0)") cmd.PersistentFlags().BoolVar(&options.verbose, "verbose", false, "Verbose mode. Print successful operations in addition to errors") cmd.PersistentFlags().StringVar(&options.route, "route", "", `Target Vespa route for feed operations (default "default")`) @@ -49,6 +50,7 @@ type feedOptions struct { speedtestBytes int speedtestSecs int waitSecs int + headers []string memprofile string cpuprofile string @@ -108,7 +110,7 @@ $ cat docs.jsonl | vespa feed -`, pprof.StartCPUProfile(f) defer pprof.StopCPUProfile() } - err := feed(args, options, cli) + err := feed(args, options, cli, cmd) if options.memprofile != "" { f, err := os.Create(options.memprofile) if err != nil { @@ -124,7 +126,7 @@ $ cat docs.jsonl | vespa feed -`, return cmd } -func createServices(n int, timeout time.Duration, waitSecs int, cli *CLI) ([]httputil.Client, string, error) { +func createServices(n int, timeout time.Duration, cli *CLI, waiter *Waiter) ([]httputil.Client, string, error) { if n < 1 { return nil, "", fmt.Errorf("need at least one client") } @@ -134,8 +136,7 @@ func createServices(n int, timeout time.Duration, waitSecs int, cli *CLI) ([]htt } services := make([]httputil.Client, 0, n) baseURL := "" - waiter := cli.waiter(time.Duration(waitSecs) * time.Second) - for i := 0; i < n; i++ { + for range n { service, err := waiter.Service(target, cli.config.cluster()) if err != nil { return nil, "", err @@ -144,7 +145,7 @@ func createServices(n int, timeout time.Duration, waitSecs int, cli *CLI) ([]htt // Create a separate HTTP client for each service client := cli.httpClientFactory(timeout) // Feeding should always use HTTP/2 - httputil.ForceHTTP2(client, service.TLSOptions.KeyPair, service.TLSOptions.CACertificate, service.TLSOptions.TrustAll) + httputil.ForceHTTP2(client, service.TLSOptions.KeyPair, service.TLSOptions.CACertificatePEM, service.TLSOptions.TrustAll) service.SetClient(client) services = append(services, service) } @@ -228,9 +229,10 @@ func enqueueAndWait(files []string, dispatcher *document.Dispatcher, options fee return fmt.Errorf("at least one file to feed from must specified") } -func feed(files []string, options feedOptions, cli *CLI) error { +func feed(files []string, options feedOptions, cli *CLI, cmd *cobra.Command) error { timeout := time.Duration(options.timeoutSecs) * time.Second - clients, baseURL, err := createServices(options.connections, timeout, options.waitSecs, cli) + waiter := cli.waiter(time.Duration(options.waitSecs)*time.Second, cmd) + clients, baseURL, err := createServices(options.connections, timeout, cli, waiter) if err != nil { return err } @@ -238,12 +240,17 @@ func feed(files []string, options feedOptions, cli *CLI) error { if err != nil { return err } + header, err := httputil.ParseHeader(options.headers) + if err != nil { + return err + } client, err := document.NewClient(document.ClientOptions{ Compression: compression, Timeout: timeout, Route: options.route, TraceLevel: options.traceLevel, BaseURL: baseURL, + Header: header, Speedtest: options.speedtestBytes > 0, NowFunc: cli.now, }, clients) diff --git a/client/go/internal/cli/cmd/feed_test.go b/client/go/internal/cli/cmd/feed_test.go index 200a0be7c5d..fc25f8e872c 100644 --- a/client/go/internal/cli/cmd/feed_test.go +++ b/client/go/internal/cli/cmd/feed_test.go @@ -82,13 +82,13 @@ func TestFeed(t *testing.T) { require.Nil(t, cli.Run("feed", "-")) assert.Equal(t, want, stdout.String()) - for i := 0; i < 10; i++ { + for range 10 { httpClient.NextResponseString(503, `{"message":"it's broken yo"}`) } require.Nil(t, cli.Run("feed", jsonFile1)) assert.Equal(t, "feed: got status 503 ({\"message\":\"it's broken yo\"}) for put id:ns:type::doc1: giving up after 10 attempts\n", stderr.String()) stderr.Reset() - for i := 0; i < 10; i++ { + for range 10 { httpClient.NextResponseError(fmt.Errorf("something else is broken")) } require.Nil(t, cli.Run("feed", jsonFile1)) diff --git a/client/go/internal/cli/cmd/fetch.go b/client/go/internal/cli/cmd/fetch.go index b2e7d11ba7b..786fe75f9c2 100644 --- a/client/go/internal/cli/cmd/fetch.go +++ b/client/go/internal/cli/cmd/fetch.go @@ -13,8 +13,7 @@ func newFetchCmd(cli *CLI) *cobra.Command { This command can be used to download an already deployed Vespa application package. The package is written as a ZIP file to the given path, or current -directory if no path is given. -`, +directory if no path is given.`, Example: `$ vespa fetch $ vespa fetch mydir/ $ vespa fetch -t cloud mycloudapp.zip diff --git a/client/go/internal/cli/cmd/log.go b/client/go/internal/cli/cmd/log.go index 77ef7f68130..53b7079f428 100644 --- a/client/go/internal/cli/cmd/log.go +++ b/client/go/internal/cli/cmd/log.go @@ -6,6 +6,7 @@ import ( "time" "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/version" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -34,7 +35,7 @@ $ vespa log --follow`, SilenceUsage: true, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - target, err := cli.target(targetOptions{logLevel: levelArg, supportedType: cloudTargetOnly}) + target, err := cli.target(targetOptions{logLevel: levelArg}) if err != nil { return err } @@ -58,7 +59,12 @@ $ vespa log --follow`, options.To = to } if err := target.PrintLog(options); err != nil { - return fmt.Errorf("could not retrieve logs: %w", err) + versionWithLogContainer := version.MustParse("8.359.0") + var hints []string + if err := target.CompatibleWith(versionWithLogContainer); err != nil { + hints = []string{fmt.Sprintf("This command requires a newer version of the Vespa platform: %s", err)} + } + return errHint(fmt.Errorf("could not retrieve logs: %w", err), hints...) } return nil }, diff --git a/client/go/internal/cli/cmd/log_test.go b/client/go/internal/cli/cmd/log_test.go index c1cab951793..e8e8a76b988 100644 --- a/client/go/internal/cli/cmd/log_test.go +++ b/client/go/internal/cli/cmd/log_test.go @@ -9,7 +9,7 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/version" ) -func TestLog(t *testing.T) { +func TestLogCloud(t *testing.T) { _, pkgDir := mock.ApplicationPackageDir(t, false, false) httpClient := &mock.HTTPClient{} httpClient.NextResponseString(200, `1632738690.905535 host1a.dev.aws-us-east-1c 806/53 logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication info Switching to the latest deployed set of configurations and components. Application config generation: 52532`) @@ -30,14 +30,13 @@ func TestLog(t *testing.T) { assert.Contains(t, stderr.String(), "Error: invalid period: cannot combine --from/--to with relative value: 1h\n") } -func TestLogOldClient(t *testing.T) { +func TestLogCloudIncompatible(t *testing.T) { cli, _, stderr := newTestCLI(t) cli.version = version.MustParse("7.0.0") _, pkgDir := mock.ApplicationPackageDir(t, false, false) httpClient := &mock.HTTPClient{} httpClient.NextResponseString(200, `{"minVersion": "8.0.0"}`) - httpClient.NextResponseString(200, `1632738690.905535 host1a.dev.aws-us-east-1c 806/53 logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication info Switching to the latest deployed set of configurations and components. Application config generation: 52532`) cli.httpClient = httpClient assert.Nil(t, cli.Run("config", "set", "application", "t1.a1.i1")) @@ -46,6 +45,37 @@ func TestLogOldClient(t *testing.T) { assert.Nil(t, cli.Run("auth", "cert", pkgDir)) assert.Nil(t, cli.Run("log")) - expected := "Warning: client version 7.0.0 is less than the minimum supported version: 8.0.0\nHint: This version may not work as expected\nHint: Try 'vespa version' to check for a new version\n" + expected := "Warning: client version 7.0.0 is less than the minimum supported version: 8.0.0\nHint: This version of CLI may not work as expected\nHint: Try 'vespa version' to check for a new version\n" assert.Contains(t, stderr.String(), expected) } + +func TestLogLocal(t *testing.T) { + httpClient := &mock.HTTPClient{} + httpClient.NextResponseString(200, `1632738690.905535 localhost 806/53 logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication info Switching to the latest deployed set of configurations and components. Application config generation: 52532`) + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = httpClient + + assert.Nil(t, cli.Run("log", "--from", "2021-09-27T10:00:00Z", "--to", "2021-09-27T11:00:00Z")) + expected := "[2021-09-27 10:31:30.905535] localhost info logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication Switching to the latest deployed set of configurations and components. Application config generation: 52532\n" + assert.Equal(t, expected, stdout.String()) + + assert.NotNil(t, cli.Run("log", "--from", "2021-09-27T13:12:49Z", "--to", "2021-09-27T13:15:00", "1h")) + assert.Contains(t, stderr.String(), "Error: invalid period: cannot combine --from/--to with relative value: 1h\n") +} + +func TestLogLocalIncompatible(t *testing.T) { + httpClient := &mock.HTTPClient{} + httpClient.NextResponseString(404, `not found`) + httpClient.NextResponse(mock.HTTPResponse{ + URI: "/state/v1/version", + Status: 200, + Body: []byte(`{"version": "8.358.0"}`), + }) + cli, _, stderr := newTestCLI(t) + cli.httpClient = httpClient + + assert.NotNil(t, cli.Run("log", "--from", "2021-09-27T10:00:00Z", "--to", "2021-09-27T11:00:00Z")) + assert.Equal(t, `Error: could not retrieve logs: failed to read logs: aborting wait: got status 404 +Hint: This command requires a newer version of the Vespa platform: platform version is older than required version: 8.358.0 < 8.359.0 +`, stderr.String()) +} diff --git a/client/go/internal/cli/cmd/prod.go b/client/go/internal/cli/cmd/prod.go index 620ec055a1d..139e4690ed2 100644 --- a/client/go/internal/cli/cmd/prod.go +++ b/client/go/internal/cli/cmd/prod.go @@ -154,7 +154,7 @@ $ vespa prod deploy`, if err := verifyTests(cli, pkg); err != nil { return err } - if err := maybeCopyCertificate(options.copyCert, true, cli, target, pkg); err != nil { + if err := requireCertificate(options.copyCert, true, cli, target, pkg); err != nil { return err } deployment := vespa.DeploymentOptions{ApplicationPackage: pkg, Target: target} @@ -430,6 +430,6 @@ func verifyTest(cli *CLI, testsParent string, suite string, required bool) error } return nil } - _, _, err = runTests(cli, testDirectory, true, 0) + _, _, err = runTests(cli, testDirectory, true, nil) return err } diff --git a/client/go/internal/cli/cmd/prod_test.go b/client/go/internal/cli/cmd/prod_test.go index e2b0b3b88de..8ea20b3bbe5 100644 --- a/client/go/internal/cli/cmd/prod_test.go +++ b/client/go/internal/cli/cmd/prod_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/vespa-engine/vespa/client/go/internal/ioutil" "github.com/vespa-engine/vespa/client/go/internal/mock" "github.com/vespa-engine/vespa/client/go/internal/vespa" @@ -157,6 +158,29 @@ func TestProdDeploy(t *testing.T) { prodDeploy(pkgDir, t) } +func TestProdDeployWithoutCertificate(t *testing.T) { + pkgDir := filepath.Join(t.TempDir(), "app") + createApplication(t, pkgDir, false, false) + + httpClient := &mock.HTTPClient{} + cli, stdout, _ := newTestCLI(t, "CI=true") + cli.httpClient = httpClient + app := vespa.ApplicationID{Tenant: "t1", Application: "a1", Instance: "i1"} + assert.Nil(t, cli.Run("config", "set", "application", app.String())) + assert.Nil(t, cli.Run("config", "set", "target", "cloud")) + assert.Nil(t, cli.Run("auth", "api-key")) + stdout.Reset() + cli.Environment["VESPA_CLI_API_KEY_FILE"] = filepath.Join(cli.config.homeDir, "t1.api-key.pem") + + // We have clients.pem, but no key pair for the application + require.Nil(t, os.MkdirAll(filepath.Join(pkgDir, "security"), 0755)) + require.Nil(t, os.WriteFile(filepath.Join(pkgDir, "security", "clients.pem"), []byte{}, 0644)) + httpClient.NextResponseString(200, `{"build": 42}`) + assert.Nil(t, cli.Run("prod", "deploy", pkgDir)) + assert.Contains(t, stdout.String(), "Success: Deployed '"+pkgDir+"' with build number 42") + assert.Contains(t, stdout.String(), "See https://console.vespa-cloud.com/tenant/t1/application/a1/prod/deployment for deployment progress") +} + func TestProdDeployWithoutTests(t *testing.T) { pkgDir := filepath.Join(t.TempDir(), "app") createApplication(t, pkgDir, false, true) diff --git a/client/go/internal/cli/cmd/query.go b/client/go/internal/cli/cmd/query.go index db6dfa0a158..5fa225777f0 100644 --- a/client/go/internal/cli/cmd/query.go +++ b/client/go/internal/cli/cmd/query.go @@ -5,7 +5,6 @@ package cmd import ( - "bufio" "encoding/json" "fmt" "io" @@ -17,6 +16,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/internal/curl" + "github.com/vespa-engine/vespa/client/go/internal/httputil" "github.com/vespa-engine/vespa/client/go/internal/ioutil" "github.com/vespa-engine/vespa/client/go/internal/sse" "github.com/vespa-engine/vespa/client/go/internal/vespa" @@ -45,7 +45,8 @@ can be set by the syntax [parameter-name]=[value].`, SilenceUsage: true, Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return query(cli, args, queryTimeoutSecs, waitSecs, printCurl, format, headers) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) + return query(cli, args, queryTimeoutSecs, printCurl, format, headers, waiter) }, } cmd.Flags().BoolVarP(&printCurl, "verbose", "v", false, "Print the equivalent curl command for the query") @@ -67,26 +68,11 @@ func printCurl(stderr io.Writer, url string, service *vespa.Service) error { return err } -func parseHeaders(headers []string) (http.Header, error) { - h := make(http.Header) - for _, header := range headers { - kv := strings.SplitN(header, ":", 2) - if len(kv) < 2 { - return nil, fmt.Errorf("invalid header %q: missing colon separator", header) - } - k := kv[0] - v := strings.TrimSpace(kv[1]) - h.Add(k, v) - } - return h, nil -} - -func query(cli *CLI, arguments []string, timeoutSecs, waitSecs int, curl bool, format string, headers []string) error { +func query(cli *CLI, arguments []string, timeoutSecs int, curl bool, format string, headers []string, waiter *Waiter) error { target, err := cli.target(targetOptions{}) if err != nil { return err } - waiter := cli.waiter(time.Duration(waitSecs) * time.Second) service, err := waiter.Service(target, cli.config.cluster()) if err != nil { return err @@ -98,7 +84,7 @@ func query(cli *CLI, arguments []string, timeoutSecs, waitSecs int, curl bool, f } url, _ := url.Parse(service.BaseURL + "/search/") urlQuery := url.Query() - for i := 0; i < len(arguments); i++ { + for i := range len(arguments) { key, value := splitArg(arguments[i]) urlQuery.Set(key, value) } @@ -118,7 +104,7 @@ func query(cli *CLI, arguments []string, timeoutSecs, waitSecs int, curl bool, f return err } } - header, err := parseHeaders(headers) + header, err := httputil.ParseHeader(headers) if err != nil { return err } @@ -159,13 +145,11 @@ type printOptions struct { func printResponseBody(body io.Reader, options printOptions, cli *CLI) error { if options.plainStream { - scanner := bufio.NewScanner(body) - for scanner.Scan() { - fmt.Fprintln(cli.Stdout, scanner.Text()) - } - return scanner.Err() + _, err := io.Copy(cli.Stdout, body) + return err } else if options.tokenStream { - dec := sse.NewDecoder(body) + bufSize := 1024 * 1024 // Handle events up to this size + dec := sse.NewDecoderSize(body, bufSize) writingLine := false for { event, err := dec.Decode() diff --git a/client/go/internal/cli/cmd/root.go b/client/go/internal/cli/cmd/root.go index 8e0f3de4f72..c0f6f3af51e 100644 --- a/client/go/internal/cli/cmd/root.go +++ b/client/go/internal/cli/cmd/root.go @@ -54,6 +54,7 @@ type CLI struct { now func() time.Time retryInterval time.Duration + waitTimeout *time.Duration cmd *cobra.Command config *Config @@ -141,7 +142,12 @@ func New(stdout, stderr io.Writer, environment []string) (*CLI, error) { Use it on Vespa instances running locally, remotely or in Vespa Cloud. -Vespa documentation: https://docs.vespa.ai +To get started, see the following quick start guides: + +- Local Vespa instance: https://docs.vespa.ai/en/vespa-quick-start.html +- Vespa Cloud: https://cloud.vespa.ai/en/getting-started + +The complete Vespa documentation is available at https://docs.vespa.ai. For detailed description of flags and configuration, see 'vespa help config'. `, @@ -153,6 +159,7 @@ For detailed description of flags and configuration, see 'vespa help config'. return fmt.Errorf("invalid command: %s", args[0]) }, } + cmd.CompletionOptions.HiddenDefaultCmd = true // Do not show the 'completion' command in help output env := make(map[string]string) for _, entry := range environment { parts := strings.SplitN(entry, "=", 2) @@ -376,7 +383,9 @@ func (c *CLI) confirm(question string, confirmByDefault bool) (bool, error) { } } -func (c *CLI) waiter(timeout time.Duration) *Waiter { return &Waiter{Timeout: timeout, cli: c} } +func (c *CLI) waiter(timeout time.Duration, cmd *cobra.Command) *Waiter { + return &Waiter{Timeout: timeout, cli: c, cmd: cmd} +} // target creates a target according the configuration of this CLI and given opts. func (c *CLI) target(opts targetOptions) (vespa.Target, error) { @@ -396,9 +405,9 @@ func (c *CLI) target(opts targetOptions) (vespa.Target, error) { if err != nil { return nil, err } - if !c.isCloudCI() { // Vespa Cloud always runs an up-to-date version - if err := target.CheckVersion(c.version); err != nil { - c.printWarning(err, "This version may not work as expected", "Try 'vespa version' to check for a new version") + if target.IsCloud() && !c.isCloudCI() { // Vespa Cloud always runs an up-to-date version + if err := target.CompatibleWith(c.version); err != nil { + c.printWarning(err, "This version of CLI may not work as expected", "Try 'vespa version' to check for a new version") } } return target, nil @@ -460,7 +469,7 @@ func (c *CLI) createCustomTarget(targetType, customURL string) (vespa.Target, er } func (c *CLI) cloudApiAuthenticator(deployment vespa.Deployment, system vespa.System) (vespa.Authenticator, error) { - apiKey, err := c.config.readAPIKey(c, system, deployment.Application.Tenant) + apiKey, err := c.config.readAPIKey(c, deployment.Application.Tenant) if err != nil { return nil, err } diff --git a/client/go/internal/cli/cmd/status.go b/client/go/internal/cli/cmd/status.go index 6056ee439b2..dff0c2d24c6 100644 --- a/client/go/internal/cli/cmd/status.go +++ b/client/go/internal/cli/cmd/status.go @@ -5,6 +5,7 @@ package cmd import ( + "errors" "fmt" "log" "strconv" @@ -49,7 +50,7 @@ $ vepsa status --format plain --cluster mycluster`, if err := verifyFormat(format); err != nil { return err } - waiter := cli.waiter(time.Duration(waitSecs) * time.Second) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) var failingContainers []*vespa.Service if cluster == "" { services, err := waiter.Services(t) @@ -125,7 +126,7 @@ func newStatusDeployCmd(cli *CLI) *cobra.Command { if err := verifyFormat(format); err != nil { return err } - waiter := cli.waiter(time.Duration(waitSecs) * time.Second) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) s, err := waiter.DeployService(t) if err != nil { return err @@ -149,7 +150,7 @@ func newStatusDeploymentCmd(cli *CLI) *cobra.Command { Long: `Show status of a Vespa deployment. This commands shows whether a Vespa deployment has converged on the latest run - (Vespa Cloud) or config generation (self-hosted). If an argument is given, +(Vespa Cloud) or config generation (self-hosted). If an argument is given, show the convergence status of that particular run or generation. `, Example: `$ vespa status deployment @@ -173,11 +174,11 @@ $ vespa status deployment -t local [session-id] --wait 600 if err != nil { return err } - waiter := cli.waiter(time.Duration(waitSecs) * time.Second) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) id, err := waiter.Deployment(t, wantedID) if err != nil { var hints []string - if waiter.Timeout == 0 { + if waiter.Timeout == 0 && !errors.Is(err, vespa.ErrDeployment) { hints = []string{"Consider using the --wait flag to wait for completion"} } return ErrCLI{Status: 1, warn: true, hints: hints, error: err} diff --git a/client/go/internal/cli/cmd/status_test.go b/client/go/internal/cli/cmd/status_test.go index 5ef96c462d8..6db27fd2778 100644 --- a/client/go/internal/cli/cmd/status_test.go +++ b/client/go/internal/cli/cmd/status_test.go @@ -85,7 +85,7 @@ func TestStatusError(t *testing.T) { cli.httpClient = client assert.NotNil(t, cli.Run("status", "container")) assert.Equal(t, - "Container default at http://127.0.0.1:8080 is not ready: unhealthy container default: status 500 at http://127.0.0.1:8080/status.html: giving up\n", + "Container default at http://127.0.0.1:8080 is not ready: unhealthy container default: status 500 at http://127.0.0.1:8080/status.html: wait deadline reached\n", stdout.String()) assert.Equal(t, "Error: services not ready: default\n", @@ -122,13 +122,13 @@ func TestStatusLocalDeployment(t *testing.T) { resp.Body = []byte(`{"currentGeneration": 42, "converged": false}`) client.NextResponse(resp) assert.NotNil(t, cli.Run("status", "deployment")) - assert.Equal(t, "Warning: deployment not converged on latest generation: giving up\nHint: Consider using the --wait flag to wait for completion\n", stderr.String()) + assert.Equal(t, "Warning: deployment not converged on latest generation: wait deadline reached\nHint: Consider using the --wait flag to wait for completion\n", stderr.String()) // Explicit generation stderr.Reset() client.NextResponse(resp) assert.NotNil(t, cli.Run("status", "deployment", "41")) - assert.Equal(t, "Warning: deployment not converged on generation 41: giving up\nHint: Consider using the --wait flag to wait for completion\n", stderr.String()) + assert.Equal(t, "Warning: deployment not converged on generation 41: wait deadline reached\nHint: Consider using the --wait flag to wait for completion\n", stderr.String()) } func TestStatusCloudDeployment(t *testing.T) { @@ -164,11 +164,11 @@ func TestStatusCloudDeployment(t *testing.T) { Body: []byte(`{"active": false, "status": "failure"}`), }) assert.NotNil(t, cli.Run("status", "deployment", "42", "-w", "10")) - assert.Equal(t, "Waiting up to 10s for deployment to converge...\nWarning: deployment run 42 incomplete after waiting up to 10s: aborting wait: run 42 ended with unsuccessful status: failure\n", stderr.String()) + assert.Equal(t, "Waiting up to 10s for deployment to converge...\nWarning: deployment run 42 not yet complete after waiting up to 10s: aborting wait: deployment failed: run 42 ended with unsuccessful status: failure\n", stderr.String()) } func isLocalTarget(args []string) bool { - for i := 0; i < len(args)-1; i++ { + for i := range len(args) - 1 { if args[i] == "-t" { return args[i+1] == "local" } @@ -197,7 +197,7 @@ func assertStatus(expectedTarget string, args []string, t *testing.T) { t.Helper() client := &mock.HTTPClient{} clusterName := "" - for i := 0; i < 3; i++ { + for range 3 { if isLocalTarget(args) { clusterName = "foo" mockServiceStatus(client, clusterName) diff --git a/client/go/internal/cli/cmd/test.go b/client/go/internal/cli/cmd/test.go index 3bc78fc91c8..5144abe95b4 100644 --- a/client/go/internal/cli/cmd/test.go +++ b/client/go/internal/cli/cmd/test.go @@ -42,7 +42,8 @@ $ vespa test src/test/application/tests/system-test/feed-and-query.json`, DisableAutoGenTag: true, SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { - count, failed, err := runTests(cli, args[0], false, waitSecs) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) + count, failed, err := runTests(cli, args[0], false, waiter) if err != nil { return err } @@ -70,7 +71,7 @@ $ vespa test src/test/application/tests/system-test/feed-and-query.json`, return testCmd } -func runTests(cli *CLI, rootPath string, dryRun bool, waitSecs int) (int, []string, error) { +func runTests(cli *CLI, rootPath string, dryRun bool, waiter *Waiter) (int, []string, error) { count := 0 failed := make([]string, 0) if stat, err := os.Stat(rootPath); err != nil { @@ -89,7 +90,7 @@ func runTests(cli *CLI, rootPath string, dryRun bool, waitSecs int) (int, []stri fmt.Fprintln(cli.Stdout, "") previousFailed = false } - failure, err := runTest(testPath, context, waitSecs) + failure, err := runTest(testPath, context, waiter) if err != nil { return 0, nil, err } @@ -101,7 +102,7 @@ func runTests(cli *CLI, rootPath string, dryRun bool, waitSecs int) (int, []stri } } } else if strings.HasSuffix(stat.Name(), ".json") { - failure, err := runTest(rootPath, testContext{testsPath: filepath.Dir(rootPath), dryRun: dryRun, cli: cli, clusters: map[string]*vespa.Service{}}, waitSecs) + failure, err := runTest(rootPath, testContext{testsPath: filepath.Dir(rootPath), dryRun: dryRun, cli: cli, clusters: map[string]*vespa.Service{}}, waiter) if err != nil { return 0, nil, err } @@ -117,7 +118,7 @@ func runTests(cli *CLI, rootPath string, dryRun bool, waitSecs int) (int, []stri } // Runs the test at the given path, and returns the specified test name if the test fails -func runTest(testPath string, context testContext, waitSecs int) (string, error) { +func runTest(testPath string, context testContext, waiter *Waiter) (string, error) { var test test testBytes, err := os.ReadFile(testPath) if err != nil { @@ -150,7 +151,7 @@ func runTest(testPath string, context testContext, waitSecs int) (string, error) if step.Name != "" { stepName += ": " + step.Name } - failure, longFailure, err := verify(step, test.Defaults.Cluster, defaultParameters, context, waitSecs) + failure, longFailure, err := verify(step, test.Defaults.Cluster, defaultParameters, context, waiter) if err != nil { fmt.Fprintln(context.cli.Stderr) return "", errHint(fmt.Errorf("error in %s: %w", stepName, err), "See https://docs.vespa.ai/en/reference/testing") @@ -173,7 +174,7 @@ func runTest(testPath string, context testContext, waitSecs int) (string, error) } // Asserts specified response is obtained for request, or returns a failure message, or an error if this fails -func verify(step step, defaultCluster string, defaultParameters map[string]string, context testContext, waitSecs int) (string, string, error) { +func verify(step step, defaultCluster string, defaultParameters map[string]string, context testContext, waiter *Waiter) (string, string, error) { requestBody, err := getBody(step.Request.BodyRaw, context.testsPath) if err != nil { return "", "", err @@ -227,9 +228,8 @@ func verify(step step, defaultCluster string, defaultParameters map[string]strin } ok := false service, ok = context.clusters[cluster] - if !ok { + if !ok && waiter != nil { // Cache service so we don't have to discover it for every step - waiter := context.cli.waiter(time.Duration(waitSecs) * time.Second) service, err = waiter.Service(target, cluster) if err != nil { return "", "", err diff --git a/client/go/internal/cli/cmd/test_test.go b/client/go/internal/cli/cmd/test_test.go index 728e8c29691..3479e057e45 100644 --- a/client/go/internal/cli/cmd/test_test.go +++ b/client/go/internal/cli/cmd/test_test.go @@ -26,11 +26,11 @@ func TestSuite(t *testing.T) { mockServiceStatus(client, "container") client.NextStatus(200) client.NextStatus(200) - for i := 0; i < 2; i++ { + for range 2 { client.NextResponseString(200, string(searchResponse)) } mockServiceStatus(client, "container") // Some tests do not specify cluster, which is fine since we only have one, but this causes a cache miss - for i := 0; i < 9; i++ { + for range 9 { client.NextResponseString(200, string(searchResponse)) } expectedBytes, _ := os.ReadFile("testdata/tests/expected-suite.out") @@ -45,7 +45,7 @@ func TestSuite(t *testing.T) { requests = append(requests, discoveryRequest) requests = append(requests, createSearchRequest(baseUrl+"/search/")) requests = append(requests, createSearchRequest(baseUrl+"/search/?foo=%2F")) - for i := 0; i < 7; i++ { + for range 7 { requests = append(requests, createSearchRequest(baseUrl+"/search/")) } assertRequests(requests, client, t) diff --git a/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/.vespaignore b/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/.vespaignore new file mode 100644 index 00000000000..24ca95c5b1e --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/.vespaignore @@ -0,0 +1,2 @@ +ignored-dir/ +ignored-file diff --git a/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/ignored-dir/file b/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/ignored-dir/file new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/ignored-dir/file @@ -0,0 +1 @@ + diff --git a/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/ignored-file b/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/ignored-file new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/ignored-file diff --git a/client/go/internal/cli/cmd/visit.go b/client/go/internal/cli/cmd/visit.go index 963833337c2..7f99df2038e 100644 --- a/client/go/internal/cli/cmd/visit.go +++ b/client/go/internal/cli/cmd/visit.go @@ -109,7 +109,8 @@ $ vespa visit --field-set "[id]" # list document IDs if !result.Success { return fmt.Errorf("argument error: %s", result.Message) } - service, err := documentService(cli, vArgs.waitSecs) + waiter := cli.waiter(time.Duration(vArgs.waitSecs)*time.Second, cmd) + service, err := documentService(cli, waiter) if err != nil { return err } diff --git a/client/go/internal/cli/cmd/visit_test.go b/client/go/internal/cli/cmd/visit_test.go index 3053b987838..85594912da2 100644 --- a/client/go/internal/cli/cmd/visit_test.go +++ b/client/go/internal/cli/cmd/visit_test.go @@ -41,7 +41,7 @@ func TestQuoteFunc(t *testing.T) { var buf []byte = make([]byte, 3) buf[0] = 'a' buf[2] = 'z' - for i := 0; i < 256; i++ { + for i := range 256 { buf[1] = byte(i) s := string(buf) res := quoteArgForUrl(s) @@ -97,7 +97,7 @@ func withMockClient(t *testing.T, prepCli func(*mock.HTTPClient), runOp func(*ve prepCli(client) cli, _, _ := newTestCLI(t) cli.httpClient = client - service, err := documentService(cli, 0) + service, err := documentService(cli, &Waiter{cli: cli}) if err != nil { t.Fatal(err) } diff --git a/client/go/internal/cli/cmd/waiter.go b/client/go/internal/cli/cmd/waiter.go index d818359e61c..8a25e18cd1e 100644 --- a/client/go/internal/cli/cmd/waiter.go +++ b/client/go/internal/cli/cmd/waiter.go @@ -2,10 +2,12 @@ package cmd import ( + "errors" "fmt" "time" "github.com/fatih/color" + "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -15,6 +17,7 @@ type Waiter struct { Timeout time.Duration // TODO(mpolden): Consider making this a budget cli *CLI + cmd *cobra.Command } // DeployService returns the service providing the deploy API on given target, @@ -81,10 +84,30 @@ func (w *Waiter) services(target vespa.Target) ([]*vespa.Service, error) { return target.ContainerServices(w.Timeout) } +// FastWaitOn returns whether we should use a short default timeout for given target. +func (w *Waiter) FastWaitOn(target vespa.Target) bool { + return target.IsCloud() && w.Timeout == 0 && !w.cmd.PersistentFlags().Changed("wait") +} + // Deployment waits for a deployment to become ready, returning the ID of the converged deployment. -func (w *Waiter) Deployment(target vespa.Target, id int64) (int64, error) { - if w.Timeout > 0 { - w.cli.printInfo("Waiting up to ", color.CyanString(w.Timeout.String()), " for deployment to converge...") +func (w *Waiter) Deployment(target vespa.Target, wantedID int64) (int64, error) { + timeout := w.Timeout + fastWait := w.FastWaitOn(target) + if timeout > 0 { + w.cli.printInfo("Waiting up to ", color.CyanString(timeout.String()), " for deployment to converge...") + } else if fastWait { + // If --wait is not explicitly given, we always wait a few seconds in Cloud to catch fast failures, e.g. + // invalid application package + timeout = 2 * time.Second + } + id, err := target.AwaitDeployment(wantedID, timeout) + if errors.Is(err, vespa.ErrWaitTimeout) { + if fastWait { + return id, nil // Do not report fast wait timeout as an error + } + if target.IsCloud() { + w.cli.printInfo("Timed out waiting for deployment to converge. See ", color.CyanString(target.Deployment().System.ConsoleRunURL(target.Deployment(), wantedID)), " for more details") + } } - return target.AwaitDeployment(id, w.Timeout) + return id, err } diff --git a/client/go/internal/httputil/httputil.go b/client/go/internal/httputil/httputil.go index e1e27de5523..56ac31d93a8 100644 --- a/client/go/internal/httputil/httputil.go +++ b/client/go/internal/httputil/httputil.go @@ -8,6 +8,7 @@ import ( "fmt" "net" "net/http" + "strings" "time" "github.com/vespa-engine/vespa/client/go/internal/build" @@ -99,3 +100,19 @@ func NewClient(timeout time.Duration) Client { }, } } + +// ParseHeader parses headers slice into a http.Header. Each element in the slice is expected to contain a string on +// the format "Header: Value". +func ParseHeader(headers []string) (http.Header, error) { + h := make(http.Header) + for _, header := range headers { + kv := strings.SplitN(header, ":", 2) + if len(kv) < 2 { + return nil, fmt.Errorf("invalid header %q: missing colon separator", header) + } + k := kv[0] + v := strings.TrimSpace(kv[1]) + h.Add(k, v) + } + return h, nil +} diff --git a/client/go/internal/mock/http.go b/client/go/internal/mock/http.go index 2fbaa85ecca..8274f4113d3 100644 --- a/client/go/internal/mock/http.go +++ b/client/go/internal/mock/http.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + "os" "strconv" "time" ) @@ -17,6 +18,9 @@ type HTTPClient struct { // The errors to return for future requests. If non-nil, these errors are returned before any responses in nextResponses. nextErrors []error + // LogRequests enables logging of all requests made through this client. + LogRequests bool + // LastRequest is the last HTTP request made through this. LastRequest *http.Request @@ -56,7 +60,12 @@ func (c *HTTPClient) NextResponseError(err error) { c.nextErrors = append(c.nextErrors, err) } +func (c *HTTPClient) Consumed() bool { return len(c.nextResponses) == 0 } + func (c *HTTPClient) Do(request *http.Request, timeout time.Duration) (*http.Response, error) { + if c.LogRequests { + fmt.Fprintf(os.Stderr, "Sending request %+v\n", request) + } c.LastRequest = request if len(c.nextErrors) > 0 { err := c.nextErrors[0] diff --git a/client/go/internal/sse/sse.go b/client/go/internal/sse/sse.go index 9a120944eec..caccc90d354 100644 --- a/client/go/internal/sse/sse.go +++ b/client/go/internal/sse/sse.go @@ -87,6 +87,15 @@ func NewDecoder(r io.Reader) *Decoder { return &Decoder{scanner: bufio.NewScanner(r)} } +// NewDecoderSize creates a new Decoder that reads from r. The size argument specifies of the size of the buffer that +// decoder will use when decoding events. Size must be large enough to fit the largest expected event. +func NewDecoderSize(r io.Reader, size int) *Decoder { + scanner := bufio.NewScanner(r) + buf := make([]byte, 0, size) + scanner.Buffer(buf, size) + return &Decoder{scanner: scanner} +} + // IsEnd returns whether this event indicates that the stream has ended. func (e Event) IsEnd() bool { return e.Name == "end" } diff --git a/client/go/internal/sse/sse_test.go b/client/go/internal/sse/sse_test.go index 0e0d6929c75..3e3decaacec 100644 --- a/client/go/internal/sse/sse_test.go +++ b/client/go/internal/sse/sse_test.go @@ -41,6 +41,13 @@ event: end assertDecodeErr(io.EOF, dec, t) } +func TestDecoderLarge(t *testing.T) { + data := strings.Repeat("c", (256*1024)-50) + r := strings.NewReader("event: foo\nid: 42\ndata: " + data + "\n") + dec := NewDecoderSize(r, 256*1024) + assertDecode(&Event{Name: "foo", ID: "42", Data: data}, dec, t) +} + func TestDecoderInvalid(t *testing.T) { r := strings.NewReader(` event: foo diff --git a/client/go/internal/vespa/application.go b/client/go/internal/vespa/application.go index a65fb41d783..d499006c982 100644 --- a/client/go/internal/vespa/application.go +++ b/client/go/internal/vespa/application.go @@ -3,6 +3,7 @@ package vespa import ( "archive/zip" + "bytes" "errors" "fmt" "io" @@ -11,6 +12,7 @@ import ( "strings" "github.com/vespa-engine/vespa/client/go/internal/ioutil" + "github.com/vespa-engine/vespa/client/go/internal/vespa/ignore" ) type ApplicationPackage struct { @@ -20,6 +22,14 @@ type ApplicationPackage struct { func (ap *ApplicationPackage) HasCertificate() bool { return ap.hasFile("security", "clients.pem") } +func (ap *ApplicationPackage) HasMatchingCertificate(certificatePEM []byte) (bool, error) { + clientsPEM, err := os.ReadFile(filepath.Join(ap.Path, "security", "clients.pem")) + if err != nil { + return false, err + } + return bytes.Equal(clientsPEM, certificatePEM), nil +} + func (ap *ApplicationPackage) HasDeploymentSpec() bool { return ap.hasFile("deployment.xml", "") } func (ap *ApplicationPackage) hasFile(pathSegment ...string) bool { @@ -73,7 +83,7 @@ func (ap *ApplicationPackage) Validate() error { func isZip(filename string) bool { return filepath.Ext(filename) == ".zip" } -func zipDir(dir string, destination string) error { +func zipDir(dir string, destination string, ignores *ignore.List) error { if !ioutil.Exists(dir) { message := "'" + dir + "' should be an application package zip or dir, but does not exist" return errors.New(message) @@ -82,22 +92,23 @@ func zipDir(dir string, destination string) error { message := "'" + dir + "' should be an application package dir, but is a (non-zip) file" return errors.New(message) } - file, err := os.Create(destination) if err != nil { message := "Could not create a temporary zip file for the application package: " + err.Error() return errors.New(message) } defer file.Close() - w := zip.NewWriter(file) defer w.Close() - walker := func(path string, info os.FileInfo, err error) error { if err != nil { return err } - if ignorePackageFile(filepath.Base(path)) { + zipPath, err := filepath.Rel(dir, path) + if err != nil { + return err + } + if ignores.Match(zipPath) { if info.IsDir() { return filepath.SkipDir } @@ -106,22 +117,16 @@ func zipDir(dir string, destination string) error { if info.IsDir() { return nil } - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - - zippath, err := filepath.Rel(dir, path) + srcFile, err := os.Open(path) if err != nil { return err } - zipfile, err := w.Create(zippath) + defer srcFile.Close() + zipFile, err := w.Create(zipPath) if err != nil { return err } - - _, err = io.Copy(zipfile, file) + _, err = io.Copy(zipFile, srcFile) if err != nil { return err } @@ -130,39 +135,38 @@ func zipDir(dir string, destination string) error { return filepath.Walk(dir, walker) } -func ignorePackageFile(name string) bool { - switch name { - case ".DS_Store": - return true +func (ap *ApplicationPackage) openZip(name string) (io.ReadCloser, error) { + f, err := os.Open(name) + if err != nil { + return nil, fmt.Errorf("could not open application package at '%s': %w", ap.Path, err) } - return false + return f, nil } func (ap *ApplicationPackage) zipReader(test bool) (io.ReadCloser, error) { - zipFile := ap.Path + path := ap.Path if test { - zipFile = ap.TestPath + path = ap.TestPath } - if !ap.IsZip() { - tempZip, err := os.CreateTemp("", "vespa") - if err != nil { - return nil, fmt.Errorf("could not create a temporary zip file for the application package: %w", err) - } - defer func() { - tempZip.Close() - os.Remove(tempZip.Name()) - // TODO: Caller must remove temporary file - }() - if err := zipDir(zipFile, tempZip.Name()); err != nil { - return nil, err - } - zipFile = tempZip.Name() + if ap.IsZip() { + return ap.openZip(path) } - f, err := os.Open(zipFile) + tmp, err := os.CreateTemp("", "vespa") if err != nil { - return nil, fmt.Errorf("could not open application package at '%s': %w", ap.Path, err) + return nil, fmt.Errorf("could not create a temporary zip file for the application package: %w", err) } - return f, nil + defer func() { + tmp.Close() + os.Remove(tmp.Name()) + }() + ignores, err := ignore.ReadFile(filepath.Join(path, ".vespaignore")) + if err != nil { + return nil, fmt.Errorf("could not read .vespaignore: %w", err) + } + if err := zipDir(path, tmp.Name(), ignores); err != nil { + return nil, err + } + return ap.openZip(tmp.Name()) } func (ap *ApplicationPackage) Unzip(test bool) (string, error) { @@ -283,7 +287,7 @@ func findApplicationPackage(zipOrDir string, options PackageOptions) (Applicatio testPath := existingPath(filepath.Join(zipOrDir, "src", "test", "application")) return ApplicationPackage{Path: path, TestPath: testPath}, nil } - // Application without Java components + // Application without Maven/Java if ioutil.Exists(filepath.Join(zipOrDir, "services.xml")) { testPath := "" if ioutil.Exists(filepath.Join(zipOrDir, "tests")) { diff --git a/client/go/internal/vespa/deploy.go b/client/go/internal/vespa/deploy.go index df74d1def8b..2c96b8b0935 100644 --- a/client/go/internal/vespa/deploy.go +++ b/client/go/internal/vespa/deploy.go @@ -7,6 +7,7 @@ package vespa import ( "bytes" "encoding/json" + "errors" "fmt" "io" "mime/multipart" @@ -20,12 +21,14 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/ioutil" "github.com/vespa-engine/vespa/client/go/internal/version" + "github.com/vespa-engine/vespa/client/go/internal/vespa/ignore" ) var ( DefaultApplication = ApplicationID{Tenant: "default", Application: "application", Instance: "default"} DefaultZone = ZoneID{Environment: "prod", Region: "default"} DefaultDeployment = Deployment{Application: DefaultApplication, Zone: DefaultZone} + ErrUnauthorized = errors.New("unauthorized") ) type ApplicationID struct { @@ -197,7 +200,7 @@ func fetchFromConfigServer(deployment DeploymentOptions, path string) error { return err } zipFile := filepath.Join(tmpDir, "application.zip") - if err := zipDir(dir, zipFile); err != nil { + if err := zipDir(dir, zipFile, &ignore.List{}); err != nil { return err } return os.Rename(zipFile, path) @@ -537,10 +540,12 @@ func uploadApplicationPackage(url *url.URL, opts DeploymentOptions) (PrepareResu } func checkResponse(req *http.Request, response *http.Response) error { - if response.StatusCode/100 == 4 { - return fmt.Errorf("invalid application package (%s)\n%s", response.Status, extractError(response.Body)) + if response.StatusCode == 401 || response.StatusCode == 403 { + return fmt.Errorf("deployment failed: %w (status %d)\n%s", ErrUnauthorized, response.StatusCode, ioutil.ReaderToJSON(response.Body)) + } else if response.StatusCode/100 == 4 { + return fmt.Errorf("invalid application package (status %d)\n%s", response.StatusCode, extractError(response.Body)) } else if response.StatusCode != 200 { - return fmt.Errorf("error from deploy API at %s (%s):\n%s", req.URL.Host, response.Status, ioutil.ReaderToJSON(response.Body)) + return fmt.Errorf("error from deploy API at %s (status %d):\n%s", req.URL.Host, response.StatusCode, ioutil.ReaderToJSON(response.Body)) } return nil } diff --git a/client/go/internal/vespa/document/document.go b/client/go/internal/vespa/document/document.go index e2a77f7b126..9c301cd7990 100644 --- a/client/go/internal/vespa/document/document.go +++ b/client/go/internal/vespa/document/document.go @@ -10,7 +10,6 @@ import ( "strconv" "strings" "sync" - "time" // Why do we use an experimental parser? This appears to be the only JSON library that satisfies the following @@ -19,7 +18,7 @@ import ( // - Supports parsing from a io.Reader // - Supports parsing token-by-token // - Few allocations during parsing (especially for large objects) - "github.com/go-json-experiment/json" + "github.com/go-json-experiment/json/jsontext" ) type Operation int @@ -29,11 +28,11 @@ const ( OperationUpdate OperationRemove - jsonArrayStart json.Kind = '[' - jsonArrayEnd json.Kind = ']' - jsonObjectStart json.Kind = '{' - jsonObjectEnd json.Kind = '}' - jsonString json.Kind = '"' + jsonArrayStart jsontext.Kind = '[' + jsonArrayEnd jsontext.Kind = ']' + jsonObjectStart jsontext.Kind = '{' + jsonObjectEnd jsontext.Kind = '}' + jsonString jsontext.Kind = '"' ) var ( @@ -153,7 +152,7 @@ func (d *Document) Reset() { // Decoder decodes documents from a JSON structure which is either an array of objects, or objects separated by newline. type Decoder struct { - dec *json.Decoder + dec *jsontext.Decoder buf bytes.Buffer array bool @@ -202,13 +201,13 @@ func (d *Decoder) guessMode() error { return nil } -func (d *Decoder) readNext(kind json.Kind) (json.Token, error) { +func (d *Decoder) readNext(kind jsontext.Kind) (jsontext.Token, error) { t, err := d.dec.ReadToken() if err != nil { - return json.Token{}, err + return jsontext.Token{}, err } if t.Kind() != kind { - return json.Token{}, fmt.Errorf("unexpected json kind: %q: want %q", t, kind) + return jsontext.Token{}, fmt.Errorf("unexpected json kind: %q: want %q", t, kind) } return t, nil } @@ -364,7 +363,7 @@ loop: func NewDecoder(r io.Reader) *Decoder { d := &Decoder{} d.documentBuffers.New = func() any { return &bytes.Buffer{} } - d.dec = json.NewDecoder(io.TeeReader(r, &d.buf)) + d.dec = jsontext.NewDecoder(io.TeeReader(r, &d.buf)) return d } diff --git a/client/go/internal/vespa/document/document_test.go b/client/go/internal/vespa/document/document_test.go index 3fcdbd3b292..8875ad83291 100644 --- a/client/go/internal/vespa/document/document_test.go +++ b/client/go/internal/vespa/document/document_test.go @@ -176,7 +176,7 @@ func testDocumentDecoder(t *testing.T, jsonLike string) { if len(docs) != len(result) { t.Errorf("len(result) = %d, want %d", len(result), len(docs)) } - for i := 0; i < len(docs); i++ { + for i := range len(docs) { got := result[i] want := docs[i] if !got.Equal(want) { @@ -206,7 +206,7 @@ func TestDocumentDecoderInvalid(t *testing.T) { t.Errorf("unexpected error: %s", err) } _, err = dec.Decode() - wantErr := "invalid operation at byte offset 110: json: invalid character '\\n' within string (expecting non-control character)" + wantErr := "invalid operation at byte offset 110: jsontext: invalid character '\\n' within string (expecting non-control character)" if err.Error() != wantErr { t.Errorf("want error %q, got %q", wantErr, err.Error()) } diff --git a/client/go/internal/vespa/document/http.go b/client/go/internal/vespa/document/http.go index 3871ab19edd..df4d97e2a82 100644 --- a/client/go/internal/vespa/document/http.go +++ b/client/go/internal/vespa/document/http.go @@ -3,6 +3,7 @@ package document import ( "bytes" + "encoding/json" "fmt" "io" "math" @@ -15,7 +16,6 @@ import ( "sync/atomic" "time" - "github.com/go-json-experiment/json" "github.com/klauspost/compress/gzip" "github.com/vespa-engine/vespa/client/go/internal/httputil" @@ -43,6 +43,7 @@ type Client struct { // ClientOptions specifices the configuration options of a feed client. type ClientOptions struct { BaseURL string + Header http.Header Timeout time.Duration Route string TraceLevel int @@ -94,48 +95,48 @@ func NewClient(options ClientOptions, httpClients []httputil.Client) (*Client, e } c.gzippers.New = func() any { return gzip.NewWriter(io.Discard) } c.buffers.New = func() any { return &bytes.Buffer{} } - for i := 0; i < runtime.NumCPU(); i++ { + for range runtime.NumCPU() { go c.preparePending() } return c, nil } -func writeQueryParam(sb *bytes.Buffer, start int, escape bool, k, v string) { - if sb.Len() == start { - sb.WriteString("?") +func writeQueryParam(buf *bytes.Buffer, start int, escape bool, k, v string) { + if buf.Len() == start { + buf.WriteString("?") } else { - sb.WriteString("&") + buf.WriteString("&") } - sb.WriteString(k) - sb.WriteString("=") + buf.WriteString(k) + buf.WriteString("=") if escape { - sb.WriteString(url.QueryEscape(v)) + buf.WriteString(url.QueryEscape(v)) } else { - sb.WriteString(v) + buf.WriteString(v) } } -func (c *Client) writeDocumentPath(id Id, sb *bytes.Buffer) { - sb.WriteString(strings.TrimSuffix(c.options.BaseURL, "/")) - sb.WriteString("/document/v1/") - sb.WriteString(url.PathEscape(id.Namespace)) - sb.WriteString("/") - sb.WriteString(url.PathEscape(id.Type)) +func (c *Client) writeDocumentPath(id Id, buf *bytes.Buffer) { + buf.WriteString(strings.TrimSuffix(c.options.BaseURL, "/")) + buf.WriteString("/document/v1/") + buf.WriteString(url.PathEscape(id.Namespace)) + buf.WriteString("/") + buf.WriteString(url.PathEscape(id.Type)) if id.Number != nil { - sb.WriteString("/number/") + buf.WriteString("/number/") n := uint64(*id.Number) - sb.WriteString(strconv.FormatUint(n, 10)) + buf.WriteString(strconv.FormatUint(n, 10)) } else if id.Group != "" { - sb.WriteString("/group/") - sb.WriteString(url.PathEscape(id.Group)) + buf.WriteString("/group/") + buf.WriteString(url.PathEscape(id.Group)) } else { - sb.WriteString("/docid") + buf.WriteString("/docid") } - sb.WriteString("/") - sb.WriteString(url.PathEscape(id.UserSpecific)) + buf.WriteString("/") + buf.WriteString(url.PathEscape(id.UserSpecific)) } -func (c *Client) methodAndURL(d Document, sb *bytes.Buffer) (string, string) { +func (c *Client) methodAndURL(d Document, buf *bytes.Buffer) (string, string) { httpMethod := "" switch d.Operation { case OperationPut: @@ -146,28 +147,28 @@ func (c *Client) methodAndURL(d Document, sb *bytes.Buffer) (string, string) { httpMethod = http.MethodDelete } // Base URL and path - c.writeDocumentPath(d.Id, sb) + c.writeDocumentPath(d.Id, buf) // Query part - queryStart := sb.Len() + queryStart := buf.Len() if c.options.Timeout > 0 { - writeQueryParam(sb, queryStart, false, "timeout", strconv.FormatInt(c.options.Timeout.Milliseconds(), 10)+"ms") + writeQueryParam(buf, queryStart, false, "timeout", strconv.FormatInt(c.options.Timeout.Milliseconds(), 10)+"ms") } if c.options.Route != "" { - writeQueryParam(sb, queryStart, true, "route", c.options.Route) + writeQueryParam(buf, queryStart, true, "route", c.options.Route) } if c.options.TraceLevel > 0 { - writeQueryParam(sb, queryStart, false, "tracelevel", strconv.Itoa(c.options.TraceLevel)) + writeQueryParam(buf, queryStart, false, "tracelevel", strconv.Itoa(c.options.TraceLevel)) } if c.options.Speedtest { - writeQueryParam(sb, queryStart, false, "dryRun", "true") + writeQueryParam(buf, queryStart, false, "dryRun", "true") } if d.Condition != "" { - writeQueryParam(sb, queryStart, true, "condition", d.Condition) + writeQueryParam(buf, queryStart, true, "condition", d.Condition) } if d.Create { - writeQueryParam(sb, queryStart, false, "create", "true") + writeQueryParam(buf, queryStart, false, "create", "true") } - return httpMethod, sb.String() + return httpMethod, buf.String() } func (c *Client) leastBusyClient() *countingHTTPClient { @@ -216,11 +217,14 @@ func (c *Client) prepare(document Document) (*http.Request, *bytes.Buffer, error return pd.request, pd.buf, pd.err } -func newRequest(method, url string, body io.Reader, gzipped bool) (*http.Request, error) { +func (c *Client) newRequest(method, url string, body io.Reader, gzipped bool) (*http.Request, error) { req, err := http.NewRequest(method, url, body) if err != nil { return nil, err } + for k, v := range c.options.Header { + req.Header[k] = v + } req.Header.Set("Content-Type", "application/json; charset=utf-8") if gzipped { req.Header.Set("Content-Encoding", "gzip") @@ -231,7 +235,7 @@ func newRequest(method, url string, body io.Reader, gzipped bool) (*http.Request func (c *Client) createRequest(method, url string, body []byte, buf *bytes.Buffer) (*http.Request, error) { buf.Reset() if len(body) == 0 { - return newRequest(method, url, nil, false) + return c.newRequest(method, url, nil, false) } useGzip := c.options.Compression == CompressionGzip || (c.options.Compression == CompressionAuto && len(body) > 512) var r io.Reader @@ -249,7 +253,7 @@ func (c *Client) createRequest(method, url string, body []byte, buf *bytes.Buffe } else { r = bytes.NewReader(body) } - return newRequest(method, url, r, useGzip) + return c.newRequest(method, url, r, useGzip) } func (c *Client) clientTimeout() time.Duration { @@ -282,11 +286,14 @@ func (c *Client) Send(document Document) Result { } // Get retrieves document with given ID. -func (c *Client) Get(id Id) Result { +func (c *Client) Get(id Id, fieldSet string) Result { start := c.now() buf := c.buffer() defer c.buffers.Put(buf) c.writeDocumentPath(id, buf) + if fieldSet != "" { + writeQueryParam(buf, buf.Len(), true, "fieldSet", fieldSet) + } url := buf.String() result := Result{Id: id} req, err := http.NewRequest(http.MethodGet, url, nil) @@ -328,7 +335,7 @@ func (c *Client) resultWithResponse(resp *http.Response, sentBytes int, result R } else { if result.Success() && c.options.TraceLevel > 0 { var jsonResponse struct { - Trace json.RawValue `json:"trace"` + Trace json.RawMessage `json:"trace"` } if err := json.Unmarshal(buf.Bytes(), &jsonResponse); err != nil { result = resultWithErr(result, fmt.Errorf("failed to decode json response: %w", err), elapsed) diff --git a/client/go/internal/vespa/document/http_test.go b/client/go/internal/vespa/document/http_test.go index 89e9e96064b..878b7a98be3 100644 --- a/client/go/internal/vespa/document/http_test.go +++ b/client/go/internal/vespa/document/http_test.go @@ -34,7 +34,7 @@ type mockHTTPClient struct { func TestLeastBusyClient(t *testing.T) { httpClient := mock.HTTPClient{} var httpClients []httputil.Client - for i := 0; i < 4; i++ { + for i := range 4 { httpClients = append(httpClients, &mockHTTPClient{i, &httpClient}) } client, _ := NewClient(ClientOptions{}, httpClients) @@ -177,7 +177,7 @@ func TestClientGet(t *testing.T) { }` id := Id{Namespace: "mynamespace", Type: "music", UserSpecific: "doc1"} httpClient.NextResponseString(200, doc) - result := client.Get(id) + result := client.Get(id, "") want := Result{ Id: id, Body: []byte(doc), @@ -189,6 +189,17 @@ func TestClientGet(t *testing.T) { if !reflect.DeepEqual(want, result) { t.Errorf("got %+v, want %+v", result, want) } + gotURL := httpClient.LastRequest.URL.String() + wantURL := "https://example.com:1337/document/v1/mynamespace/music/docid/doc1" + if gotURL != wantURL { + t.Errorf("got URL=%s, want %s", gotURL, wantURL) + } + client.Get(id, "[all]") + gotURL = httpClient.LastRequest.URL.String() + wantURL = "https://example.com:1337/document/v1/mynamespace/music/docid/doc1?fieldSet=%5Ball%5D" + if gotURL != wantURL { + t.Errorf("got URL=%s, want %s", gotURL, wantURL) + } } func TestClientSendCompressed(t *testing.T) { diff --git a/client/go/internal/vespa/document/throttler_test.go b/client/go/internal/vespa/document/throttler_test.go index eba8cbd2972..3cdecb22be4 100644 --- a/client/go/internal/vespa/document/throttler_test.go +++ b/client/go/internal/vespa/document/throttler_test.go @@ -13,7 +13,7 @@ func TestThrottler(t *testing.T) { if got, want := tr.TargetInflight(), int64(16); got != want { t.Errorf("got TargetInflight() = %d, but want %d", got, want) } - for i := 0; i < 65; i++ { + for range 65 { tr.Sent() tr.Success() } diff --git a/client/go/internal/vespa/ignore/ignore.go b/client/go/internal/vespa/ignore/ignore.go new file mode 100644 index 00000000000..ea045ec326b --- /dev/null +++ b/client/go/internal/vespa/ignore/ignore.go @@ -0,0 +1,62 @@ +package ignore + +import ( + "bufio" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +// List is a list of ignore patterns. +type List struct{ patterns []string } + +// Match returns whether path matches any pattern in this list. +func (l *List) Match(path string) bool { + for _, pattern := range l.patterns { + if ok, _ := filepath.Match(pattern, path); ok { + return true + } + // A directory exclude applies to all subpaths + if strings.HasSuffix(pattern, string(filepath.Separator)) && strings.HasPrefix(path, pattern) { + return true + } + } + return false +} + +// Read reads an ignore list from reader r. +func Read(r io.Reader) (*List, error) { + scanner := bufio.NewScanner(r) + ignore := List{} + line := 0 + for scanner.Scan() { + line++ + pattern := strings.TrimSpace(scanner.Text()) + if pattern == "" || strings.HasPrefix(pattern, "#") { + continue + } + if _, err := filepath.Match(pattern, ""); err != nil { + return nil, fmt.Errorf("line %d: bad pattern: %s: %w", line, pattern, err) + } + ignore.patterns = append(ignore.patterns, pattern) + } + if err := scanner.Err(); err != nil { + return nil, err + } + return &ignore, nil +} + +// ReadFile reads an ignore list from the named file. Reading a non-existent file returns an empty list, and no error. +func ReadFile(name string) (*List, error) { + f, err := os.Open(name) + if err != nil { + if os.IsNotExist(err) { + return &List{}, nil + } + return nil, err + } + defer f.Close() + return Read(f) +} diff --git a/client/go/internal/vespa/ignore/ignore_test.go b/client/go/internal/vespa/ignore/ignore_test.go new file mode 100644 index 00000000000..79ed437cc2d --- /dev/null +++ b/client/go/internal/vespa/ignore/ignore_test.go @@ -0,0 +1,50 @@ +package ignore + +import ( + "strings" + "testing" +) + +func TestRead(t *testing.T) { + f := ` +# files + +foo +foob* +???.tmp + +# directories + + foo/bar +foo/*/baz +bar/ +` + list, err := Read(strings.NewReader(f)) + if err != nil { + t.Fatal(err) + } + assertMatch(t, list, "", false) + assertMatch(t, list, "\n", false) + assertMatch(t, list, "foo1", false) + assertMatch(t, list, "foo", true) + assertMatch(t, list, "foobar", true) + assertMatch(t, list, "foo/bar", true) + assertMatch(t, list, "foo/bar/baz", true) + assertMatch(t, list, "foo/bar/bax", false) + assertMatch(t, list, "bar", false) + assertMatch(t, list, "bar/", true) + assertMatch(t, list, "bar/x", true) + assertMatch(t, list, "foo.tmp", true) + assertMatch(t, list, "fooo.tmp", false) + + _, err = Read(strings.NewReader("myfile[")) + if err == nil { + t.Fatal("want error") + } +} + +func assertMatch(t *testing.T, list *List, name string, match bool) { + if got := list.Match(name); got != match { + t.Errorf("Match(%q) = %t, want %t", name, got, match) + } +} diff --git a/client/go/internal/vespa/load_env.go b/client/go/internal/vespa/load_env.go index 5cae03694bc..4eb1297e711 100644 --- a/client/go/internal/vespa/load_env.go +++ b/client/go/internal/vespa/load_env.go @@ -151,7 +151,7 @@ func nSpacedFields(s string, n int) []string { // pretty strict for now, can be more lenient if needed func isValidShellVariableName(s string) bool { - for i := 0; i < len(s); i++ { + for i := range len(s) { b := s[i] switch { case (b >= 'A' && b <= 'Z'): // ok diff --git a/client/go/internal/vespa/target.go b/client/go/internal/vespa/target.go index 94eb2cbafe4..674bedc9343 100644 --- a/client/go/internal/vespa/target.go +++ b/client/go/internal/vespa/target.go @@ -3,11 +3,14 @@ package vespa import ( + "bytes" "crypto/tls" "errors" "fmt" "io" + "math" "net/http" + "strconv" "strings" "time" @@ -36,9 +39,15 @@ const ( AnyDeployment int64 = -2 ) -var errWaitTimeout = errors.New("giving up") var errAuth = errors.New("auth failed") +var ( + // ErrWaitTimeout is the error returned when waiting for something times out. + ErrWaitTimeout = errors.New("wait deadline reached") + // ErrDeployment is the error returned for terminal deployment failures. + ErrDeployment = errors.New("deployment failed") +) + // Authenticator authenticates the given HTTP request. type Authenticator interface { Authenticate(request *http.Request) error @@ -114,15 +123,18 @@ type Target interface { // PrintLog writes the logs of this deployment using given options to control output. PrintLog(options LogOptions) error - // CheckVersion verifies whether clientVersion is compatible with this target. - CheckVersion(clientVersion version.Version) error + // CompatibleWith returns nil if target is compatible with the given version. + CompatibleWith(version version.Version) error } // TLSOptions holds the client certificate to use for cloud API or service requests. type TLSOptions struct { - CACertificate []byte - KeyPair []tls.Certificate - TrustAll bool + KeyPair []tls.Certificate + TrustAll bool + + CACertificatePEM []byte + CertificatePEM []byte + PrivateKeyPEM []byte CACertificateFile string CertificateFile string @@ -143,7 +155,7 @@ type LogOptions struct { func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Response, error) { if !s.customClient { // Do not override TLS config if a custom client has been configured - httputil.ConfigureTLS(s.httpClient, s.TLSOptions.KeyPair, s.TLSOptions.CACertificate, s.TLSOptions.TrustAll) + httputil.ConfigureTLS(s.httpClient, s.TLSOptions.KeyPair, s.TLSOptions.CACertificatePEM, s.TLSOptions.TrustAll) } if s.auth != nil { if err := s.auth.Authenticate(request); err != nil { @@ -243,6 +255,64 @@ func isOK(status int) (bool, error) { } } +func deployServiceWait(target Target, fn responseFunc, reqFn requestFunc, timeout, retryInterval time.Duration) (int, error) { + deployService, err := target.DeployService() + if err != nil { + return 0, err + } + return wait(deployService, fn, reqFn, timeout, retryInterval) +} + +func pollLogs(target Target, logsURL string, options LogOptions, retryInterval time.Duration) error { + req, err := http.NewRequest("GET", logsURL, nil) + if err != nil { + return err + } + lastFrom := options.From + requestFunc := func() *http.Request { + fromMillis := lastFrom.Unix() * 1000 + q := req.URL.Query() + q.Set("from", strconv.FormatInt(fromMillis, 10)) + if !options.To.IsZero() { + toMillis := options.To.Unix() * 1000 + q.Set("to", strconv.FormatInt(toMillis, 10)) + } + req.URL.RawQuery = q.Encode() + return req + } + logFunc := func(status int, response []byte) (bool, error) { + if ok, err := isOK(status); !ok { + return ok, err + } + logEntries, err := ReadLogEntries(bytes.NewReader(response)) + if err != nil { + return false, err + } + for _, le := range logEntries { + if !le.Time.After(lastFrom) { + continue + } + if LogLevel(le.Level) > options.Level { + continue + } + fmt.Fprintln(options.Writer, le.Format(options.Dequote)) + } + if len(logEntries) > 0 { + lastFrom = logEntries[len(logEntries)-1].Time + } + return false, nil + } + var timeout time.Duration + if options.Follow { + timeout = math.MaxInt64 // No timeout + } + // Ignore wait error because logFunc has no concept of completion, we just want to print log entries until timeout is reached + if _, err := deployServiceWait(target, logFunc, requestFunc, timeout, retryInterval); err != nil && !errors.Is(err, ErrWaitTimeout) { + return fmt.Errorf("failed to read logs: %s", err) + } + return nil +} + // responseFunc returns whether a HTTP request is considered successful, based on its status and response data. // Returning false indicates that the operation should be retried. An error is returned if the response is considered // terminal and that the request should not be retried. @@ -290,7 +360,7 @@ func wait(service *Service, okFn responseFunc, reqFn requestFunc, timeout, retry time.Sleep(retryInterval) } if err == nil { - return status, errWaitTimeout + return status, ErrWaitTimeout } return status, err } diff --git a/client/go/internal/vespa/target_cloud.go b/client/go/internal/vespa/target_cloud.go index c063b99edef..05d6bdd224e 100644 --- a/client/go/internal/vespa/target_cloud.go +++ b/client/go/internal/vespa/target_cloud.go @@ -2,11 +2,8 @@ package vespa import ( - "bytes" "encoding/json" - "errors" "fmt" - "math" "net/http" "sort" "strconv" @@ -148,7 +145,7 @@ func (t *cloudTarget) ContainerServices(timeout time.Duration) ([]*Service, erro return services, nil } -func (t *cloudTarget) CheckVersion(clientVersion version.Version) error { +func (t *cloudTarget) CompatibleWith(clientVersion version.Version) error { if clientVersion.IsZero() { // development version is always fine return nil } @@ -190,61 +187,7 @@ func (t *cloudTarget) logsURL() string { } func (t *cloudTarget) PrintLog(options LogOptions) error { - req, err := http.NewRequest("GET", t.logsURL(), nil) - if err != nil { - return err - } - lastFrom := options.From - requestFunc := func() *http.Request { - fromMillis := lastFrom.Unix() * 1000 - q := req.URL.Query() - q.Set("from", strconv.FormatInt(fromMillis, 10)) - if !options.To.IsZero() { - toMillis := options.To.Unix() * 1000 - q.Set("to", strconv.FormatInt(toMillis, 10)) - } - req.URL.RawQuery = q.Encode() - return req - } - logFunc := func(status int, response []byte) (bool, error) { - if ok, err := isOK(status); !ok { - return ok, err - } - logEntries, err := ReadLogEntries(bytes.NewReader(response)) - if err != nil { - return false, err - } - for _, le := range logEntries { - if !le.Time.After(lastFrom) { - continue - } - if LogLevel(le.Level) > options.Level { - continue - } - fmt.Fprintln(options.Writer, le.Format(options.Dequote)) - } - if len(logEntries) > 0 { - lastFrom = logEntries[len(logEntries)-1].Time - } - return false, nil - } - var timeout time.Duration - if options.Follow { - timeout = math.MaxInt64 // No timeout - } - // Ignore wait error because logFunc has no concept of completion, we just want to print log entries until timeout is reached - if _, err := t.deployServiceWait(logFunc, requestFunc, timeout); err != nil && !errors.Is(err, errWaitTimeout) { - return fmt.Errorf("failed to read logs: %s", err) - } - return nil -} - -func (t *cloudTarget) deployServiceWait(fn responseFunc, reqFn requestFunc, timeout time.Duration) (int, error) { - deployService, err := t.DeployService() - if err != nil { - return 0, err - } - return wait(deployService, fn, reqFn, timeout, t.retryInterval) + return pollLogs(t, t.logsURL(), options, t.retryInterval) } func (t *cloudTarget) discoverLatestRun(timeout time.Duration) (int64, error) { @@ -269,7 +212,7 @@ func (t *cloudTarget) discoverLatestRun(timeout time.Duration) (int64, error) { } return false, nil } - _, err = t.deployServiceWait(jobsSuccessFunc, requestFunc, timeout) + _, err = deployServiceWait(t, jobsSuccessFunc, requestFunc, timeout, t.retryInterval) return lastRunID, err } @@ -309,17 +252,17 @@ func (t *cloudTarget) AwaitDeployment(runID int64, timeout time.Duration) (int64 return false, nil } if resp.Status != "success" { - return false, fmt.Errorf("run %d ended with unsuccessful status: %s", runID, resp.Status) + return false, fmt.Errorf("%w: run %d ended with unsuccessful status: %s", ErrDeployment, runID, resp.Status) } success = true return success, nil } - _, err = t.deployServiceWait(jobSuccessFunc, requestFunc, timeout) + _, err = deployServiceWait(t, jobSuccessFunc, requestFunc, timeout, t.retryInterval) if err != nil { - return 0, fmt.Errorf("deployment run %d incomplete%s: %w", runID, waitDescription(timeout), err) + return 0, fmt.Errorf("deployment run %d not yet complete%s: %w", runID, waitDescription(timeout), err) } if !success { - return 0, fmt.Errorf("deployment run %d incomplete%s", runID, waitDescription(timeout)) + return 0, fmt.Errorf("deployment run %d not yet complete%s", runID, waitDescription(timeout)) } return runID, nil } @@ -378,7 +321,7 @@ func (t *cloudTarget) discoverEndpoints(timeout time.Duration) (map[string]strin } return true, nil } - if _, err := t.deployServiceWait(endpointFunc, func() *http.Request { return req }, timeout); err != nil { + if _, err := deployServiceWait(t, endpointFunc, func() *http.Request { return req }, timeout, t.retryInterval); err != nil { return nil, fmt.Errorf("no endpoints found in zone %s%s: %w", t.deploymentOptions.Deployment.Zone, waitDescription(timeout), err) } if len(urlsByCluster) == 0 { diff --git a/client/go/internal/vespa/target_custom.go b/client/go/internal/vespa/target_custom.go index 9d62f7dc297..1f72308178a 100644 --- a/client/go/internal/vespa/target_custom.go +++ b/client/go/internal/vespa/target_custom.go @@ -64,10 +64,48 @@ func (t *customTarget) IsCloud() bool { return false } func (t *customTarget) Deployment() Deployment { return DefaultDeployment } func (t *customTarget) PrintLog(options LogOptions) error { - return fmt.Errorf("log access is only supported on cloud: run vespa-logfmt on the admin node instead, or export from a container image (here named 'vespa') using docker exec vespa vespa-logfmt") + deployService, err := t.DeployService() + if err != nil { + return err + } + logsURL := deployService.BaseURL + "/application/v2/tenant/default/application/default/environment/prod/region/default/instance/default/logs" + return pollLogs(t, logsURL, options, t.retryInterval) } -func (t *customTarget) CheckVersion(version version.Version) error { return nil } +func (t *customTarget) CompatibleWith(minVersion version.Version) error { + if minVersion.IsZero() { // development version is always fine + return nil + } + deployService, err := t.DeployService() + if err != nil { + return err + } + versionURL := deployService.BaseURL + "/state/v1/version" + req, err := http.NewRequest("GET", versionURL, nil) + if err != nil { + return err + } + var versionResponse struct { + Version string `json:"version"` + } + response, err := deployService.Do(req, 10*time.Second) + if err != nil { + return err + } + defer response.Body.Close() + dec := json.NewDecoder(response.Body) + if err := dec.Decode(&versionResponse); err != nil { + return err + } + targetVersion, err := version.Parse(versionResponse.Version) + if err != nil { + return err + } + if targetVersion.Less(minVersion) { + return fmt.Errorf("platform version is older than required version: %s < %s", targetVersion, minVersion) + } + return nil +} func (t *customTarget) newService(url, name string, deployAPI bool) *Service { return &Service{ diff --git a/client/go/internal/vespa/target_test.go b/client/go/internal/vespa/target_test.go index 4c2fda8368e..2b4cf485b83 100644 --- a/client/go/internal/vespa/target_test.go +++ b/client/go/internal/vespa/target_test.go @@ -20,7 +20,7 @@ func TestLocalTarget(t *testing.T) { client := &mock.HTTPClient{} lt := LocalTarget(client, TLSOptions{}, 0) assertServiceURL(t, "http://127.0.0.1:19071", lt, "deploy") - for i := 0; i < 2; i++ { + for range 2 { response := ` { "services": [ @@ -83,7 +83,7 @@ func TestCustomTargetWait(t *testing.T) { client.NextStatus(500) assertService(t, true, target, "", 0) // Fails multiple times - for i := 0; i < 3; i++ { + for range 3 { client.NextStatus(500) client.NextResponseError(io.EOF) } @@ -117,6 +117,22 @@ func TestCustomTargetAwaitDeployment(t *testing.T) { assert.Equal(t, int64(42), convergedID) } +func TestCustomTargetCompatibleWith(t *testing.T) { + client := &mock.HTTPClient{} + target := CustomTarget(client, "http://192.0.2.42", TLSOptions{}, 0) + for range 3 { + client.NextResponse(mock.HTTPResponse{ + URI: "/state/v1/version", + Status: 200, + Body: []byte(`{"version": "1.2.3"}`), + }) + } + assert.Nil(t, target.CompatibleWith(version.MustParse("1.2.2"))) + assert.Nil(t, target.CompatibleWith(version.MustParse("1.2.3"))) + assert.NotNil(t, target.CompatibleWith(version.MustParse("1.2.4"))) + assert.True(t, client.Consumed()) +} + func TestCloudTargetWait(t *testing.T) { var logWriter bytes.Buffer target, client := createCloudTarget(t, &logWriter) @@ -231,14 +247,14 @@ func TestLog(t *testing.T) { assert.Equal(t, expected, buf.String()) } -func TestCheckVersion(t *testing.T) { +func TestCloudCompatibleWith(t *testing.T) { target, client := createCloudTarget(t, io.Discard) - for i := 0; i < 3; i++ { + for range 3 { client.NextResponse(mock.HTTPResponse{URI: "/cli/v1/", Status: 200, Body: []byte(`{"minVersion":"8.0.0"}`)}) } - assert.Nil(t, target.CheckVersion(version.MustParse("8.0.0"))) - assert.Nil(t, target.CheckVersion(version.MustParse("8.1.0"))) - assert.NotNil(t, target.CheckVersion(version.MustParse("7.0.0"))) + assert.Nil(t, target.CompatibleWith(version.MustParse("8.0.0"))) + assert.Nil(t, target.CompatibleWith(version.MustParse("8.1.0"))) + assert.NotNil(t, target.CompatibleWith(version.MustParse("7.0.0"))) } func createCloudTarget(t *testing.T, logWriter io.Writer) (Target, *mock.HTTPClient) { |