diff options
303 files changed, 4518 insertions, 7138 deletions
diff --git a/application/src/main/java/com/yahoo/application/container/DocumentProcessing.java b/application/src/main/java/com/yahoo/application/container/DocumentProcessing.java index f89e79c4cfa..f0f85fb78dd 100644 --- a/application/src/main/java/com/yahoo/application/container/DocumentProcessing.java +++ b/application/src/main/java/com/yahoo/application/container/DocumentProcessing.java @@ -62,8 +62,10 @@ public final class DocumentProcessing { @SuppressWarnings("removal") // TODO Vespa 8: remove public DocumentProcessor.Progress process(ComponentSpecification chain, com.yahoo.docproc.Processing processing) { DocprocExecutor executor = getExecutor(chain); - // TODO Vespa 8: Use TBD instead, this method will be removed + + // TODO Vespa 8: Remove statement (registry will be removed from Processing) processing.setDocprocServiceRegistry(handler.getDocprocServiceRegistry()); + return executor.processUntilDone(processing); } @@ -83,8 +85,10 @@ public final class DocumentProcessing { @SuppressWarnings("removal") // TODO Vespa 8: remove public DocumentProcessor.Progress processOnce(ComponentSpecification chain, com.yahoo.docproc.Processing processing) { DocprocExecutor executor = getExecutor(chain); - // TODO Vespa 8: Use TBD instead, this method will be removed + + // TODO Vespa 8: Remove statement (registry will be removed from Processing) processing.setDocprocServiceRegistry(handler.getDocprocServiceRegistry()); + return executor.process(processing); } diff --git a/client/go/.dir-locals.el b/client/go/.dir-locals.el new file mode 100644 index 00000000000..1c7c82921be --- /dev/null +++ b/client/go/.dir-locals.el @@ -0,0 +1,3 @@ +((go-mode + . ((eglot-workspace-configuration + . ((:gopls . ((staticcheck . t)))))))) diff --git a/client/go/Makefile b/client/go/Makefile index 1d995c7f5bf..ce182b5d653 100644 --- a/client/go/Makefile +++ b/client/go/Makefile @@ -31,6 +31,10 @@ all: test checkfmt install # Bump the version of the vespa-cli formula and create a pull request to Homebrew repository. # +# Homebrew's automated BrewTestBot must do the merge for the bottles to be published. When +# the PR has been merged check that the merge was done by their bot, and that installing +# the new version works by running: `brew update && brew install vespa-cli`. +# # Example: # # $ git checkout vX.Y.Z @@ -106,6 +110,9 @@ install-all: all manpages # install: +ifdef CI + go env +endif env GOBIN=$(BIN) $(GOPROXY_OVERRIDE) go install $(GO_FLAGS) ./... manpages: install diff --git a/client/go/auth0/auth0.go b/client/go/auth/auth0/auth0.go index 2f7040b3d37..52ba3f085a4 100644 --- a/client/go/auth0/auth0.go +++ b/client/go/auth/auth0/auth0.go @@ -166,13 +166,7 @@ func (a *Auth0) PrepareSystem(ctx context.Context) (*System, error) { res, err := tr.Refresh(ctx, a.system) if err != nil { - // ask and guide the user through the login process: - fmt.Println(fmt.Errorf("failed to renew access token, %s", err)) - fmt.Print("\n") - s, err = RunLogin(ctx, a, true) - if err != nil { - return nil, err - } + return nil, fmt.Errorf("failed to renew access token: %w: %s", err, "re-authenticate with 'vespa auth login'") } else { // persist the updated system with renewed access token s.AccessToken = res.AccessToken diff --git a/client/go/zts/zts.go b/client/go/auth/zts/zts.go index 538971ebdd8..d288c2050d9 100644 --- a/client/go/zts/zts.go +++ b/client/go/auth/zts/zts.go @@ -16,12 +16,12 @@ const DefaultURL = "https://zts.athenz.ouroath.com:4443" // Client is a client for Athenz ZTS, an authentication token service. type Client struct { - client util.HttpClient + client util.HTTPClient tokenURL *url.URL } // NewClient creates a new client for an Athenz ZTS service located at serviceURL. -func NewClient(serviceURL string, client util.HttpClient) (*Client, error) { +func NewClient(serviceURL string, client util.HTTPClient) (*Client, error) { tokenURL, err := url.Parse(serviceURL) if err != nil { return nil, err diff --git a/client/go/zts/zts_test.go b/client/go/auth/zts/zts_test.go index 0eec085aadb..0eec085aadb 100644 --- a/client/go/zts/zts_test.go +++ b/client/go/auth/zts/zts_test.go diff --git a/client/go/cmd/api_key.go b/client/go/cmd/api_key.go index c6b6ae1d1e3..5e23cdc9ebd 100644 --- a/client/go/cmd/api_key.go +++ b/client/go/cmd/api_key.go @@ -14,9 +14,12 @@ import ( "github.com/vespa-engine/vespa/client/go/vespa" ) -var overwriteKey bool - -const apiKeyLongDoc = `Create a new user API key for authentication with Vespa Cloud. +func newAPIKeyCmd(cli *CLI, deprecated bool) *cobra.Command { + var overwriteKey bool + cmd := &cobra.Command{ + Use: "api-key", + Short: "Create a new user API key for authentication with Vespa Cloud", + Long: `Create a new user API key for authentication with Vespa Cloud. The API key will be stored in the Vespa CLI home directory (see 'vespa help config'). Other commands will then automatically load the API @@ -34,65 +37,40 @@ can be useful in continuous integration systems. export VESPA_CLI_API_KEY_FILE=/path/to/api-key Note that when overriding API key through environment variables, that key will -always be used. It's not possible to specify a tenant-specific key.` - -func init() { - apiKeyCmd.Flags().BoolVarP(&overwriteKey, "force", "f", false, "Force overwrite of existing API key") - apiKeyCmd.MarkPersistentFlagRequired(applicationFlag) - deprecatedApiKeyCmd.Flags().BoolVarP(&overwriteKey, "force", "f", false, "Force overwrite of existing API key") - deprecatedApiKeyCmd.MarkPersistentFlagRequired(applicationFlag) -} - -func apiKeyExample() string { - return "$ vespa auth api-key -a my-tenant.my-app.my-instance" -} - -var apiKeyCmd = &cobra.Command{ - Use: "api-key", - Short: "Create a new user API key for authentication with Vespa Cloud", - Long: apiKeyLongDoc, - Example: apiKeyExample(), - DisableAutoGenTag: true, - SilenceUsage: true, - Args: cobra.ExactArgs(0), - RunE: doApiKey, -} - -// TODO: Remove this after 2022-06-01 -var deprecatedApiKeyCmd = &cobra.Command{ - Use: "api-key", - Short: "Create a new user API key for authentication with Vespa Cloud", - Long: apiKeyLongDoc, - Example: apiKeyExample(), - DisableAutoGenTag: true, - SilenceUsage: true, - Args: cobra.ExactArgs(0), - Hidden: true, - Deprecated: "use 'vespa auth api-key' instead", - RunE: doApiKey, +always be used. It's not possible to specify a tenant-specific key.`, + Example: "$ vespa auth api-key -a my-tenant.my-app.my-instance", + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + return doApiKey(cli, overwriteKey, args) + }, + } + if deprecated { + cmd.Deprecated = "use 'vespa auth api-key' instead" + } + cmd.Flags().BoolVarP(&overwriteKey, "force", "f", false, "Force overwrite of existing API key") + cmd.MarkPersistentFlagRequired(applicationFlag) + return cmd } -func doApiKey(_ *cobra.Command, _ []string) error { - cfg, err := LoadConfig() - if err != nil { - return fmt.Errorf("could not load config: %w", err) - } - app, err := getApplication() +func doApiKey(cli *CLI, overwriteKey bool, args []string) error { + app, err := cli.config.application() if err != nil { return err } - targetType, err := getTargetType() + targetType, err := cli.config.targetType() if err != nil { return err } - system, err := getSystem(targetType) + system, err := cli.system(targetType) if err != nil { return err } - apiKeyFile := cfg.APIKeyPath(app.Tenant) + apiKeyFile := cli.config.apiKeyPath(app.Tenant) if util.PathExists(apiKeyFile) && !overwriteKey { err := fmt.Errorf("refusing to overwrite %s", apiKeyFile) - printErrHint(err, "Use -f to overwrite it") + cli.printErrHint(err, "Use -f to overwrite it") printPublicKey(system, apiKeyFile, app.Tenant) return ErrCLI{error: err, quiet: true} } @@ -101,7 +79,7 @@ func doApiKey(_ *cobra.Command, _ []string) error { return fmt.Errorf("could not create api key: %w", err) } if err := ioutil.WriteFile(apiKeyFile, apiKey, 0600); err == nil { - printSuccess("API private key written to ", apiKeyFile) + cli.printSuccess("API private key written to ", apiKeyFile) return printPublicKey(system, apiKeyFile, app.Tenant) } else { return fmt.Errorf("failed to write: %s: %w", apiKeyFile, err) diff --git a/client/go/cmd/api_key_test.go b/client/go/cmd/api_key_test.go index ba697b69d9f..3e3a7fa0a31 100644 --- a/client/go/cmd/api_key_test.go +++ b/client/go/cmd/api_key_test.go @@ -4,7 +4,6 @@ package cmd import ( - "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -20,16 +19,20 @@ func TestAPIKey(t *testing.T) { } func testAPIKey(t *testing.T, subcommand []string) { - homeDir := filepath.Join(t.TempDir(), ".vespa") - keyFile := filepath.Join(homeDir, "t1.api-key.pem") + cli, stdout, stderr := newTestCLI(t) - execute(command{args: []string{"config", "set", "target", "cloud"}, homeDir: homeDir}, t, nil) + err := cli.Run("config", "set", "target", "cloud") + assert.Nil(t, err) args := append(subcommand, "-a", "t1.a1.i1") - out, _ := execute(command{args: args, homeDir: homeDir}, t, nil) - assert.Contains(t, out, "Success: API private key written to "+keyFile+"\n") - - out, outErr := execute(command{args: args, homeDir: homeDir}, t, nil) - assert.Contains(t, outErr, "Error: refusing to overwrite "+keyFile+"\nHint: Use -f to overwrite it\n") - assert.Contains(t, out, "This is your public key") + err = cli.Run(args...) + assert.Nil(t, err) + assert.Equal(t, "", stderr.String()) + assert.Contains(t, stdout.String(), "Success: API private key written to") + + err = cli.Run(subcommand...) + assert.NotNil(t, err) + assert.Contains(t, stderr.String(), "Error: refusing to overwrite") + assert.Contains(t, stderr.String(), "Hint: Use -f to overwrite it\n") + assert.Contains(t, stdout.String(), "This is your public key") } diff --git a/client/go/cmd/auth.go b/client/go/cmd/auth.go index 592c18c8b22..453d2296b08 100644 --- a/client/go/cmd/auth.go +++ b/client/go/cmd/auth.go @@ -6,24 +6,16 @@ import ( "github.com/spf13/cobra" ) -func init() { - rootCmd.AddCommand(authCmd) - rootCmd.AddCommand(deprecatedCertCmd) - rootCmd.AddCommand(deprecatedApiKeyCmd) - authCmd.AddCommand(certCmd) - authCmd.AddCommand(apiKeyCmd) - authCmd.AddCommand(loginCmd) - authCmd.AddCommand(logoutCmd) -} - -var authCmd = &cobra.Command{ - Use: "auth", - Short: "Manage Vespa Cloud credentials", - Long: `Manage Vespa Cloud credentials.`, - DisableAutoGenTag: true, - SilenceUsage: false, - Args: cobra.MinimumNArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - return fmt.Errorf("invalid command: %s", args[0]) - }, +func newAuthCmd() *cobra.Command { + return &cobra.Command{ + Use: "auth", + Short: "Manage Vespa Cloud credentials", + Long: `Manage Vespa Cloud credentials.`, + DisableAutoGenTag: true, + SilenceUsage: false, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return fmt.Errorf("invalid command: %s", args[0]) + }, + } } diff --git a/client/go/cmd/cert.go b/client/go/cmd/cert.go index 672442b9ad9..caa32b0a963 100644 --- a/client/go/cmd/cert.go +++ b/client/go/cmd/cert.go @@ -16,25 +16,15 @@ import ( "github.com/vespa-engine/vespa/client/go/vespa" ) -var ( - noApplicationPackage bool - overwriteCertificate bool -) - -func init() { - certCmd.Flags().BoolVarP(&overwriteCertificate, "force", "f", false, "Force overwrite of existing certificate and private key") - certCmd.Flags().BoolVarP(&noApplicationPackage, "no-add", "N", false, "Do not add certificate to the application package") - certCmd.MarkPersistentFlagRequired(applicationFlag) - - deprecatedCertCmd.Flags().BoolVarP(&overwriteCertificate, "force", "f", false, "Force overwrite of existing certificate and private key") - deprecatedCertCmd.MarkPersistentFlagRequired(applicationFlag) - - certCmd.AddCommand(certAddCmd) - certAddCmd.Flags().BoolVarP(&overwriteCertificate, "force", "f", false, "Force overwrite of existing certificate") - certAddCmd.MarkPersistentFlagRequired(applicationFlag) -} - -var longDoc = `Create a new private key and self-signed certificate for Vespa Cloud deployment. +func newCertCmd(cli *CLI, deprecated bool) *cobra.Command { + var ( + noApplicationPackage bool + overwriteCertificate bool + ) + cmd := &cobra.Command{ + Use: "cert", + Short: "Create a new private key and self-signed certificate for Vespa Cloud deployment", + Long: `Create a new private key and self-signed certificate for Vespa Cloud deployment. The private key and certificate will be stored in the Vespa CLI home directory (see 'vespa help config'). Other commands will then automatically load the @@ -56,54 +46,55 @@ Example of loading certificate and key from custom paths: Note that when overriding key pair through environment variables, that key pair will always be used for all applications. It's not possible to specify an -application-specific key.` - -var certCmd = &cobra.Command{ - Use: "cert", - Short: "Create a new private key and self-signed certificate for Vespa Cloud deployment", - Long: longDoc, - Example: `$ vespa auth cert -a my-tenant.my-app.my-instance +application-specific key.`, + Example: `$ vespa auth cert -a my-tenant.my-app.my-instance $ vespa auth cert -a my-tenant.my-app.my-instance path/to/application/package`, - DisableAutoGenTag: true, - SilenceUsage: true, - Args: cobra.MaximumNArgs(1), - RunE: doCert, -} - -// TODO: Remove this after 2022-06-01 -var deprecatedCertCmd = &cobra.Command{ - Use: "cert", - Short: "Create a new private key and self-signed certificate for Vespa Cloud deployment", - Long: longDoc, - Example: "$ vespa cert -a my-tenant.my-app.my-instance", - DisableAutoGenTag: true, - SilenceUsage: true, - Args: cobra.MaximumNArgs(1), - Deprecated: "use 'vespa auth cert' instead", - Hidden: true, - RunE: doCert, + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return doCert(cli, overwriteCertificate, noApplicationPackage, args) + }, + } + cmd.Flags().BoolVarP(&overwriteCertificate, "force", "f", false, "Force overwrite of existing certificate and private key") + if deprecated { + // TODO: Remove this after 2022-06-01 + cmd.Deprecated = "use 'vespa auth cert' instead" + } else { + cmd.Flags().BoolVarP(&noApplicationPackage, "no-add", "N", false, "Do not add certificate to the application package") + } + cmd.MarkPersistentFlagRequired(applicationFlag) + return cmd } -var certAddCmd = &cobra.Command{ - Use: "add", - Short: "Add certificate to application package", - Long: `Add an existing self-signed certificate for Vespa Cloud deployment to your application package. +func newCertAddCmd(cli *CLI) *cobra.Command { + var overwriteCertificate bool + cmd := &cobra.Command{ + Use: "add", + Short: "Add certificate to application package", + Long: `Add an existing self-signed certificate for Vespa Cloud deployment to your application package. The certificate will be loaded from the Vespa CLI home directory (see 'vespa help config') by default. The location of the application package can be specified as an argument to this command (default '.').`, - Example: `$ vespa auth cert add -a my-tenant.my-app.my-instance + Example: `$ vespa auth cert add -a my-tenant.my-app.my-instance $ vespa auth cert add -a my-tenant.my-app.my-instance path/to/application/package`, - DisableAutoGenTag: true, - SilenceUsage: true, - Args: cobra.MaximumNArgs(1), - RunE: doCertAdd, + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return doCertAdd(cli, overwriteCertificate, args) + }, + } + cmd.Flags().BoolVarP(&overwriteCertificate, "force", "f", false, "Force overwrite of existing certificate") + cmd.MarkPersistentFlagRequired(applicationFlag) + return cmd } -func doCert(_ *cobra.Command, args []string) error { - app, err := getApplication() +func doCert(cli *CLI, overwriteCertificate, noApplicationPackage bool, args []string) error { + app, err := cli.config.application() if err != nil { return err } @@ -114,15 +105,11 @@ func doCert(_ *cobra.Command, args []string) error { return err } } - cfg, err := LoadConfig() - if err != nil { - return err - } - privateKeyFile, err := cfg.PrivateKeyPath(app) + privateKeyFile, err := cli.config.privateKeyPath(app) if err != nil { return err } - certificateFile, err := cfg.CertificatePath(app) + certificateFile, err := cli.config.certificatePath(app) if err != nil { return err } @@ -169,15 +156,15 @@ func doCert(_ *cobra.Command, args []string) error { return fmt.Errorf("could not write private key: %w", err) } if !noApplicationPackage { - printSuccess("Certificate written to ", color.CyanString(pkgCertificateFile)) + cli.printSuccess("Certificate written to ", color.CyanString(pkgCertificateFile)) } - printSuccess("Certificate written to ", color.CyanString(certificateFile)) - printSuccess("Private key written to ", color.CyanString(privateKeyFile)) + cli.printSuccess("Certificate written to ", color.CyanString(certificateFile)) + cli.printSuccess("Private key written to ", color.CyanString(privateKeyFile)) return nil } -func doCertAdd(_ *cobra.Command, args []string) error { - app, err := getApplication() +func doCertAdd(cli *CLI, overwriteCertificate bool, args []string) error { + app, err := cli.config.application() if err != nil { return err } @@ -185,11 +172,7 @@ func doCertAdd(_ *cobra.Command, args []string) error { if err != nil { return err } - cfg, err := LoadConfig() - if err != nil { - return err - } - certificateFile, err := cfg.CertificatePath(app) + certificateFile, err := cli.config.certificatePath(app) if err != nil { return err } @@ -226,6 +209,6 @@ func doCertAdd(_ *cobra.Command, args []string) error { return fmt.Errorf("could not copy certificate file to application: %w", err) } - printSuccess("Certificate written to ", color.CyanString(pkgCertificateFile)) + cli.printSuccess("Certificate written to ", color.CyanString(pkgCertificateFile)) return nil } diff --git a/client/go/cmd/cert_test.go b/client/go/cmd/cert_test.go index 51e99938d2b..e5837170d15 100644 --- a/client/go/cmd/cert_test.go +++ b/client/go/cmd/cert_test.go @@ -25,24 +25,28 @@ func TestCert(t *testing.T) { } func testCert(t *testing.T, subcommand []string) { - homeDir := filepath.Join(t.TempDir(), ".vespa") pkgDir := mockApplicationPackage(t, false) + + cli, stdout, stderr := newTestCLI(t) args := append(subcommand, "-a", "t1.a1.i1", pkgDir) - out, _ := execute(command{args: args, homeDir: homeDir}, t, nil) + err := cli.Run(args...) + assert.Nil(t, err) app, err := vespa.ApplicationFromString("t1.a1.i1") assert.Nil(t, err) appDir := filepath.Join(pkgDir, "src", "main", "application") pkgCertificate := filepath.Join(appDir, "security", "clients.pem") + homeDir := cli.config.homeDir certificate := filepath.Join(homeDir, app.String(), "data-plane-public-cert.pem") privateKey := filepath.Join(homeDir, app.String(), "data-plane-private-key.pem") - assert.Equal(t, fmt.Sprintf("Success: Certificate written to %s\nSuccess: Certificate written to %s\nSuccess: Private key written to %s\n", pkgCertificate, certificate, privateKey), out) + assert.Equal(t, fmt.Sprintf("Success: Certificate written to %s\nSuccess: Certificate written to %s\nSuccess: Private key written to %s\n", pkgCertificate, certificate, privateKey), stdout.String()) args = append(subcommand, "-a", "t1.a1.i1", pkgDir) - _, outErr := execute(command{args: args, homeDir: homeDir}, t, nil) - assert.Contains(t, outErr, fmt.Sprintf("Error: application package %s already contains a certificate", appDir)) + err = cli.Run(args...) + assert.NotNil(t, err) + assert.Contains(t, stderr.String(), fmt.Sprintf("Error: application package %s already contains a certificate", appDir)) } func TestCertCompressedPackage(t *testing.T) { @@ -55,7 +59,6 @@ func TestCertCompressedPackage(t *testing.T) { } func testCertCompressedPackage(t *testing.T, subcommand []string) { - homeDir := filepath.Join(t.TempDir(), ".vespa") pkgDir := mockApplicationPackage(t, true) zipFile := filepath.Join(pkgDir, "target", "application.zip") err := os.MkdirAll(filepath.Dir(zipFile), 0755) @@ -63,71 +66,71 @@ func testCertCompressedPackage(t *testing.T, subcommand []string) { _, err = os.Create(zipFile) assert.Nil(t, err) + cli, stdout, stderr := newTestCLI(t) + args := append(subcommand, "-a", "t1.a1.i1", pkgDir) - _, outErr := execute(command{args: args, homeDir: homeDir}, t, nil) - assert.Contains(t, outErr, "Error: cannot add certificate to compressed application package") + err = cli.Run(args...) + assert.NotNil(t, err) + assert.Contains(t, stderr.String(), "Error: cannot add certificate to compressed application package") err = os.Remove(zipFile) assert.Nil(t, err) args = append(subcommand, "-f", "-a", "t1.a1.i1", pkgDir) - out, _ := execute(command{args: args, homeDir: homeDir}, t, nil) - assert.Contains(t, out, "Success: Certificate written to") - assert.Contains(t, out, "Success: Private key written to") + err = cli.Run(args...) + assert.Nil(t, err) + assert.Contains(t, stdout.String(), "Success: Certificate written to") + assert.Contains(t, stdout.String(), "Success: Private key written to") } func TestCertAdd(t *testing.T) { - homeDir := filepath.Join(t.TempDir(), ".vespa") - execute(command{args: []string{"auth", "cert", "-N", "-a", "t1.a1.i1"}, homeDir: homeDir}, t, nil) + cli, stdout, stderr := newTestCLI(t) + err := cli.Run("auth", "cert", "-N", "-a", "t1.a1.i1") + assert.Nil(t, err) pkgDir := mockApplicationPackage(t, false) - out, _ := execute(command{args: []string{"auth", "cert", "add", "-a", "t1.a1.i1", pkgDir}, homeDir: homeDir}, t, nil) + stdout.Reset() + err = cli.Run("auth", "cert", "add", "-a", "t1.a1.i1", pkgDir) + assert.Nil(t, err) appDir := filepath.Join(pkgDir, "src", "main", "application") pkgCertificate := filepath.Join(appDir, "security", "clients.pem") - assert.Equal(t, fmt.Sprintf("Success: Certificate written to %s\n", pkgCertificate), out) + assert.Equal(t, fmt.Sprintf("Success: Certificate written to %s\n", pkgCertificate), stdout.String()) - out, outErr := execute(command{args: []string{"auth", "cert", "add", "-a", "t1.a1.i1", pkgDir}, homeDir: homeDir}, t, nil) - assert.Equal(t, "", out) - assert.Contains(t, outErr, fmt.Sprintf("Error: application package %s already contains a certificate", appDir)) - out, _ = execute(command{args: []string{"auth", "cert", "add", "-f", "-a", "t1.a1.i1", pkgDir}, homeDir: homeDir}, t, nil) - assert.Equal(t, fmt.Sprintf("Success: Certificate written to %s\n", pkgCertificate), out) + err = cli.Run("auth", "cert", "add", "-a", "t1.a1.i1", pkgDir) + assert.NotNil(t, err) + assert.Contains(t, stderr.String(), fmt.Sprintf("Error: application package %s already contains a certificate", appDir)) + stdout.Reset() + err = cli.Run("auth", "cert", "add", "-f", "-a", "t1.a1.i1", pkgDir) + assert.Nil(t, err) + assert.Equal(t, fmt.Sprintf("Success: Certificate written to %s\n", pkgCertificate), stdout.String()) } func TestCertNoAdd(t *testing.T) { - homeDir := filepath.Join(t.TempDir(), ".vespa") - out, _ := execute(command{args: []string{"auth", "cert", "-N", "-a", "t1.a1.i1"}, homeDir: homeDir}, t, nil) + cli, stdout, stderr := newTestCLI(t) + + err := cli.Run("auth", "cert", "-N", "-a", "t1.a1.i1") + assert.Nil(t, err) + homeDir := cli.config.homeDir app, err := vespa.ApplicationFromString("t1.a1.i1") assert.Nil(t, err) certificate := filepath.Join(homeDir, app.String(), "data-plane-public-cert.pem") privateKey := filepath.Join(homeDir, app.String(), "data-plane-private-key.pem") - assert.Equal(t, fmt.Sprintf("Success: Certificate written to %s\nSuccess: Private key written to %s\n", certificate, privateKey), out) + assert.Equal(t, fmt.Sprintf("Success: Certificate written to %s\nSuccess: Private key written to %s\n", certificate, privateKey), stdout.String()) - _, outErr := execute(command{args: []string{"auth", "cert", "-N", "-a", "t1.a1.i1"}, homeDir: homeDir}, t, nil) - assert.Contains(t, outErr, fmt.Sprintf("Error: private key %s already exists", privateKey)) + err = cli.Run("auth", "cert", "-N", "-a", "t1.a1.i1") + assert.NotNil(t, err) + assert.Contains(t, stderr.String(), fmt.Sprintf("Error: private key %s already exists", privateKey)) require.Nil(t, os.Remove(privateKey)) - _, outErr = execute(command{args: []string{"auth", "cert", "-N", "-a", "t1.a1.i1"}, homeDir: homeDir}, t, nil) - assert.Contains(t, outErr, fmt.Sprintf("Error: certificate %s already exists", certificate)) - out, _ = execute(command{args: []string{"auth", "cert", "-N", "-f", "-a", "t1.a1.i1"}, homeDir: homeDir}, t, nil) - assert.Equal(t, fmt.Sprintf("Success: Certificate written to %s\nSuccess: Private key written to %s\n", certificate, privateKey), out) -} + stderr.Reset() + err = cli.Run("auth", "cert", "-N", "-a", "t1.a1.i1") + assert.NotNil(t, err) + assert.Contains(t, stderr.String(), fmt.Sprintf("Error: certificate %s already exists", certificate)) -func mockApplicationPackage(t *testing.T, java bool) string { - dir := t.TempDir() - appDir := filepath.Join(dir, "src", "main", "application") - if err := os.MkdirAll(appDir, 0755); err != nil { - t.Fatal(err) - } - servicesXML := filepath.Join(appDir, "services.xml") - if _, err := os.Create(servicesXML); err != nil { - t.Fatal(err) - } - if java { - if _, err := os.Create(filepath.Join(dir, "pom.xml")); err != nil { - t.Fatal(err) - } - } - return dir + stdout.Reset() + err = cli.Run("auth", "cert", "-N", "-f", "-a", "t1.a1.i1") + assert.Nil(t, err) + assert.Equal(t, fmt.Sprintf("Success: Certificate written to %s\nSuccess: Private key written to %s\n", certificate, privateKey), stdout.String()) } diff --git a/client/go/cmd/clone.go b/client/go/cmd/clone.go index a1add299712..180ec18debf 100644 --- a/client/go/cmd/clone.go +++ b/client/go/cmd/clone.go @@ -22,24 +22,15 @@ import ( "github.com/vespa-engine/vespa/client/go/util" ) -const ( - sampleAppsCacheTTL = time.Hour * 168 // 1 week - sampleAppsFilename = "sample-apps-master.zip" -) - -var listApps bool -var forceClone bool - -func init() { - rootCmd.AddCommand(cloneCmd) - cloneCmd.Flags().BoolVarP(&listApps, "list", "l", false, "List available sample applications") - cloneCmd.Flags().BoolVarP(&forceClone, "force", "f", false, "Ignore cache and force downloading the latest sample application from GitHub") -} - -var cloneCmd = &cobra.Command{ - Use: "clone sample-application-path target-directory", - Short: "Create files and directory structure for a new Vespa application from a sample application", - Long: `Create files and directory structure for a new Vespa application +func newCloneCmd(cli *CLI) *cobra.Command { + var ( + listApps bool + forceClone bool + ) + cmd := &cobra.Command{ + Use: "clone sample-application-path target-directory", + Short: "Create files and directory structure for a new Vespa application from a sample application", + Long: `Create files and directory structure for a new Vespa application from a sample application. Sample applications are downloaded from @@ -48,29 +39,33 @@ https://github.com/vespa-engine/sample-apps. By default sample applications are cached in the user's cache directory. This directory can be overriden by setting the VESPA_CLI_CACHE_DIR environment variable.`, - Example: "$ vespa clone vespa-cloud/album-recommendation my-app", - DisableAutoGenTag: true, - SilenceUsage: true, - RunE: func(cmd *cobra.Command, args []string) error { - if listApps { - apps, err := listSampleApps() - if err != nil { - return fmt.Errorf("could not list sample applications: %w", err) + Example: "$ vespa clone vespa-cloud/album-recommendation my-app", + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + if listApps { + apps, err := listSampleApps(cli.httpClient) + if err != nil { + return fmt.Errorf("could not list sample applications: %w", err) + } + for _, app := range apps { + log.Print(app) + } + return nil } - for _, app := range apps { - log.Print(app) + if len(args) != 2 { + return fmt.Errorf("expected exactly 2 arguments, got %d", len(args)) } - return nil - } - if len(args) != 2 { - return fmt.Errorf("expected exactly 2 arguments, got %d", len(args)) - } - return cloneApplication(args[0], args[1]) - }, + return cloneApplication(cli, args[0], args[1], forceClone) + }, + } + cmd.Flags().BoolVarP(&listApps, "list", "l", false, "List available sample applications") + cmd.Flags().BoolVarP(&forceClone, "force", "f", false, "Ignore cache and force downloading the latest sample application from GitHub") + return cmd } -func cloneApplication(applicationName string, applicationDir string) error { - zipFile, err := openSampleAppsZip() +func cloneApplication(cli *CLI, applicationName string, applicationDir string, force bool) error { + zipFile, err := openSampleAppsZip(force, cli) if err != nil { return err } @@ -107,26 +102,26 @@ func cloneApplication(applicationName string, applicationDir string) error { return nil } -func useCache(stat os.FileInfo) (bool, error) { - if forceClone { +func useCache(force bool, stat os.FileInfo) (bool, error) { + if force { return false, nil } - expiry := stat.ModTime().Add(sampleAppsCacheTTL) + expiry := stat.ModTime().Add(time.Hour * 168) // 1 week return stat.Size() > 0 && time.Now().Before(expiry), nil } -func fetchSampleAppsZip(destination string) error { +func fetchSampleAppsZip(destination string, cli *CLI) error { f, err := ioutil.TempFile(filepath.Dir(destination), "sample-apps") if err != nil { return fmt.Errorf("could not create temporary file: %w", err) } defer f.Close() - return util.Spinner(stderr, color.YellowString("Downloading sample apps ..."), func() error { + return util.Spinner(cli.Stderr, color.YellowString("Downloading sample apps ..."), func() error { request, err := http.NewRequest("GET", "https://github.com/vespa-engine/sample-apps/archive/refs/heads/master.zip", nil) if err != nil { return fmt.Errorf("invalid url: %w", err) } - response, err := util.HttpDo(request, time.Minute*60, "GitHub") + response, err := cli.httpClient.Do(request, time.Minute*60) if err != nil { return fmt.Errorf("could not download sample apps: %w", err) } @@ -145,12 +140,12 @@ func fetchSampleAppsZip(destination string) error { }) } -func openSampleAppsZip() (*os.File, error) { - cacheDir, err := vespaCliCacheDir() +func openSampleAppsZip(forceClone bool, cli *CLI) (*os.File, error) { + cacheDir, err := vespaCliCacheDir(cli.Environment) if err != nil { return nil, err } - path := filepath.Join(cacheDir, sampleAppsFilename) + path := filepath.Join(cacheDir, "sample-apps-master.zip") cacheExists := true stat, err := os.Stat(path) if errors.Is(err, os.ErrNotExist) { @@ -159,7 +154,7 @@ func openSampleAppsZip() (*os.File, error) { return nil, fmt.Errorf("could not stat existing cache file: %w", err) } if cacheExists { - useCache, err := useCache(stat) + useCache, err := useCache(forceClone, stat) if err != nil { return nil, errHint(fmt.Errorf("could not determine cache status: %w", err), "Try ignoring the cache with the -f flag") } @@ -168,7 +163,7 @@ func openSampleAppsZip() (*os.File, error) { return os.Open(path) } } - if err := fetchSampleAppsZip(path); err != nil { + if err := fetchSampleAppsZip(path, cli); err != nil { return nil, fmt.Errorf("could not fetch sample apps: %w", err) } return os.Open(path) diff --git a/client/go/cmd/clone_list.go b/client/go/cmd/clone_list.go index 83eb3fdc62f..1d37c0a8617 100644 --- a/client/go/cmd/clone_list.go +++ b/client/go/cmd/clone_list.go @@ -10,12 +10,12 @@ import ( "github.com/vespa-engine/vespa/client/go/util" ) -func listSampleApps() ([]string, error) { - return listSampleAppsAt("https://api.github.com/repos/vespa-engine/sample-apps/contents/") +func listSampleApps(client util.HTTPClient) ([]string, error) { + return listSampleAppsAt("https://api.github.com/repos/vespa-engine/sample-apps/contents/", client) } -func listSampleAppsAt(url string) ([]string, error) { - rfs, err := getRepositoryFiles(url) +func listSampleAppsAt(url string, client util.HTTPClient) ([]string, error) { + rfs, err := getRepositoryFiles(url, client) if err != nil { return nil, err } @@ -25,7 +25,7 @@ func listSampleAppsAt(url string) ([]string, error) { if isApp { apps = append(apps, rf.Path) } else if follow { - apps2, err := listSampleAppsAt(rf.URL) + apps2, err := listSampleAppsAt(rf.URL, client) if err != nil { return nil, err } @@ -36,12 +36,12 @@ func listSampleAppsAt(url string) ([]string, error) { return apps, nil } -func getRepositoryFiles(url string) ([]repositoryFile, error) { +func getRepositoryFiles(url string, client util.HTTPClient) ([]repositoryFile, error) { req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } - response, err := util.HttpDo(req, time.Minute, "GitHub") + response, err := client.Do(req, time.Minute) if err != nil { return nil, err } diff --git a/client/go/cmd/clone_list_test.go b/client/go/cmd/clone_list_test.go index 2e4fc4004bd..73aa70f12f2 100644 --- a/client/go/cmd/clone_list_test.go +++ b/client/go/cmd/clone_list_test.go @@ -8,7 +8,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/vespa-engine/vespa/client/go/mock" - "github.com/vespa-engine/vespa/client/go/util" ) func TestListSampleApps(t *testing.T) { @@ -17,9 +16,8 @@ func TestListSampleApps(t *testing.T) { c.NextResponse(200, readTestData(t, "sample-apps-news.json")) c.NextResponse(200, readTestData(t, "sample-apps-operations.json")) c.NextResponse(200, readTestData(t, "sample-apps-vespa-cloud.json")) - util.ActiveHttpClient = c - apps, err := listSampleApps() + apps, err := listSampleApps(c) assert.Nil(t, err) expected := []string{ "album-recommendation-monitoring", diff --git a/client/go/cmd/clone_test.go b/client/go/cmd/clone_test.go index 332758a127a..9587a1435d3 100644 --- a/client/go/cmd/clone_test.go +++ b/client/go/cmd/clone_test.go @@ -22,23 +22,26 @@ func TestClone(t *testing.T) { func assertCreated(sampleAppName string, app string, t *testing.T) { appCached := app + "-cache" + defer os.RemoveAll(app) + defer os.RemoveAll(appCached) + httpClient := &mock.HTTPClient{} testdata, err := ioutil.ReadFile(filepath.Join("testdata", "sample-apps-master.zip")) require.Nil(t, err) httpClient.NextResponseBytes(200, testdata) - cacheDir := t.TempDir() - require.Nil(t, err) - defer func() { - os.RemoveAll(cacheDir) - }() - out, _ := execute(command{failTestOnError: true, cacheDir: cacheDir, args: []string{"clone", sampleAppName, app}}, t, httpClient) - defer os.RemoveAll(app) - assert.Equal(t, "Created "+app+"\n", out) + + cli, stdout, _ := newTestCLI(t) + cli.httpClient = httpClient + err = cli.Run("clone", sampleAppName, app) + assert.Nil(t, err) + + assert.Equal(t, "Created "+app+"\n", stdout.String()) assertFiles(t, app) - outCached, _ := execute(command{failTestOnError: true, cacheDir: cacheDir, args: []string{"clone", sampleAppName, appCached}}, t, nil) - defer os.RemoveAll(appCached) - assert.Equal(t, "Using cached sample apps ...\nCreated "+appCached+"\n", outCached) + stdout.Reset() + err = cli.Run("clone", sampleAppName, appCached) + assert.Nil(t, err) + assert.Equal(t, "Using cached sample apps ...\nCreated "+appCached+"\n", stdout.String()) assertFiles(t, appCached) } diff --git a/client/go/cmd/command_tester_test.go b/client/go/cmd/command_tester_test.go deleted file mode 100644 index efaca6a7258..00000000000 --- a/client/go/cmd/command_tester_test.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// A helper for testing commands -// Author: bratseth - -package cmd - -import ( - "bytes" - "io" - "os" - "path/filepath" - "testing" - - "github.com/spf13/pflag" - "github.com/spf13/viper" - "github.com/stretchr/testify/require" - "github.com/vespa-engine/vespa/client/go/mock" - "github.com/vespa-engine/vespa/client/go/util" -) - -type command struct { - homeDir string - cacheDir string - stdin io.ReadWriter - args []string - moreArgs []string - env map[string]string - failTestOnError bool -} - -func resetFlag(f *pflag.Flag) { - switch v := f.Value.(type) { - case pflag.SliceValue: - _ = v.Replace([]string{}) - default: - switch v.Type() { - case "bool", "string", "int": - _ = v.Set(f.DefValue) - } - } -} - -func setEnv(env map[string]string) map[string]string { - originalEnv := map[string]string{} - for k, v := range env { - value, ok := os.LookupEnv(k) - if ok { - originalEnv[k] = value - } - os.Setenv(k, v) - } - return originalEnv -} - -func resetEnv(env map[string]string, original map[string]string) { - for k := range env { - if v, ok := original[k]; ok { - os.Setenv(k, v) - } else { - os.Unsetenv(k) - } - } -} - -func execute(cmd command, t *testing.T, client *mock.HTTPClient) (string, string) { - if client != nil { - util.ActiveHttpClient = client - } - - // Set Vespa CLI directories. Use a separate one per test if none is specified - if cmd.homeDir == "" { - cmd.homeDir = filepath.Join(t.TempDir(), ".vespa") - viper.Reset() - } - if cmd.cacheDir == "" { - cmd.cacheDir = filepath.Join(t.TempDir(), ".cache", "vespa") - } - - env := map[string]string{} - for k, v := range cmd.env { - env[k] = v - } - env["VESPA_CLI_HOME"] = cmd.homeDir - env["VESPA_CLI_CACHE_DIR"] = cmd.cacheDir - originalEnv := setEnv(env) - defer resetEnv(env, originalEnv) - - // Reset viper at end of test to ensure vespa config set does not leak between tests - t.Cleanup(viper.Reset) - - // Reset flags to their default value - persistent flags in Cobra persists over tests - // TODO: Due to the bad design of viper, the only proper fix is to get rid of global state by moving each command to - // their own sub-package - rootCmd.Flags().VisitAll(resetFlag) - queryCmd.Flags().VisitAll(resetFlag) - documentCmd.Flags().VisitAll(resetFlag) - logCmd.Flags().VisitAll(resetFlag) - certCmd.Flags().VisitAll(resetFlag) - certAddCmd.Flags().VisitAll(resetFlag) - - // Capture stdout and execute command - var capturedOut bytes.Buffer - var capturedErr bytes.Buffer - stdout = &capturedOut - stderr = &capturedErr - if cmd.stdin != nil { - stdin = cmd.stdin - } else { - stdin = os.Stdin - } - - // Execute command and return output - rootCmd.SetArgs(append(cmd.args, cmd.moreArgs...)) - err := Execute() - if cmd.failTestOnError { - require.Nil(t, err) - } - return capturedOut.String(), capturedErr.String() -} - -func executeCommand(t *testing.T, client *mock.HTTPClient, args []string, moreArgs []string) string { - out, _ := execute(command{args: args, moreArgs: moreArgs}, t, client) - return out -} diff --git a/client/go/cmd/config.go b/client/go/cmd/config.go index 59739d1c342..75bd9959280 100644 --- a/client/go/cmd/config.go +++ b/client/go/cmd/config.go @@ -18,7 +18,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" "github.com/spf13/viper" - "github.com/vespa-engine/vespa/client/go/auth0" + "github.com/vespa-engine/vespa/client/go/auth/auth0" "github.com/vespa-engine/vespa/client/go/util" "github.com/vespa-engine/vespa/client/go/vespa" ) @@ -28,87 +28,81 @@ const ( configType = "yaml" ) -var ( - flagToConfigBindings map[string]*cobra.Command = make(map[string]*cobra.Command) - envToConfigBindings map[string]string = make(map[string]string) -) - -func init() { - rootCmd.AddCommand(configCmd) - configCmd.AddCommand(setConfigCmd) - configCmd.AddCommand(getConfigCmd) -} - -var configCmd = &cobra.Command{ - Use: "config", - Short: "Configure persistent values for flags", - Long: `Configure persistent values for flags. +func newConfigCmd() *cobra.Command { + return &cobra.Command{ + Use: "config", + Short: "Configure persistent values for global flags", + Long: `Configure persistent values for global flags. -This command allows setting a persistent value for a given flag. On future -invocations the flag can then be omitted as it is read from the config file -instead. +This command allows setting a persistent value for a given global flag. On +future invocations the flag can then be omitted as it is read from the config +file instead. Configuration is written to $HOME/.vespa by default. This path can be overridden by setting the VESPA_CLI_HOME environment variable.`, - DisableAutoGenTag: true, - SilenceUsage: false, - Args: cobra.MinimumNArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - return fmt.Errorf("invalid command: %s", args[0]) - }, + DisableAutoGenTag: true, + SilenceUsage: false, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return fmt.Errorf("invalid command: %s", args[0]) + }, + } } -var setConfigCmd = &cobra.Command{ - Use: "set option-name value", - Short: "Set a configuration option.", - Example: "$ vespa config set target cloud", - DisableAutoGenTag: true, - SilenceUsage: true, - Args: cobra.ExactArgs(2), - RunE: func(cmd *cobra.Command, args []string) error { - cfg, err := LoadConfig() - if err != nil { - return err - } - if err := cfg.Set(args[0], args[1]); err != nil { - return err - } - return cfg.Write() - }, +func newConfigSetCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "set option-name value", + Short: "Set a configuration option.", + Example: "$ vespa config set target cloud", + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + if err := cli.config.set(args[0], args[1]); err != nil { + return err + } + return cli.config.write() + }, + } } -var getConfigCmd = &cobra.Command{ - Use: "get [option-name]", - Short: "Show given configuration option, or all configuration options", - Example: `$ vespa config get +func newConfigGetCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "get [option-name]", + Short: "Show given configuration option, or all configuration options", + Example: `$ vespa config get $ vespa config get target`, - Args: cobra.MaximumNArgs(1), - DisableAutoGenTag: true, - SilenceUsage: true, - RunE: func(cmd *cobra.Command, args []string) error { - cfg, err := LoadConfig() - if err != nil { - return err - } - if len(args) == 0 { // Print all values - var flags []string - for flag := range flagToConfigBindings { - flags = append(flags, flag) - } - sort.Strings(flags) - for _, flag := range flags { - printOption(cfg, flag) + Args: cobra.MaximumNArgs(1), + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { // Print all values + var flags []string + for flag := range cli.config.bindings.flag { + flags = append(flags, flag) + } + sort.Strings(flags) + for _, flag := range flags { + cli.config.printOption(flag) + } + } else { + cli.config.printOption(args[0]) } - } else { - printOption(cfg, args[0]) - } - return nil - }, + return nil + }, + } } type Config struct { - Home string - createDirs bool + homeDir string + environment map[string]string + bindings ConfigBindings + createDirs bool +} + +type ConfigBindings struct { + flag map[string]*cobra.Command + environment map[string]string } type KeyPair struct { @@ -117,23 +111,43 @@ type KeyPair struct { PrivateKeyFile string } -func LoadConfig() (*Config, error) { - home, err := vespaCliHome() +func NewConfigBindings() ConfigBindings { + return ConfigBindings{ + flag: make(map[string]*cobra.Command), + environment: make(map[string]string), + } +} + +func (b *ConfigBindings) bindFlag(name string, command *cobra.Command) { + b.flag[name] = command +} + +func (b *ConfigBindings) bindEnvironment(flagName string, variable string) { + b.environment[flagName] = variable +} + +func loadConfig(environment map[string]string, bindings ConfigBindings) (*Config, error) { + home, err := vespaCliHome(environment) if err != nil { return nil, fmt.Errorf("could not detect config directory: %w", err) } - c := &Config{Home: home, createDirs: true} + c := &Config{ + homeDir: home, + environment: environment, + bindings: bindings, + createDirs: true, + } if err := c.load(); err != nil { return nil, fmt.Errorf("could not load config: %w", err) } return c, nil } -func (c *Config) Write() error { - if err := os.MkdirAll(c.Home, 0700); err != nil { +func (c *Config) write() error { + if err := os.MkdirAll(c.homeDir, 0700); err != nil { return err } - configFile := filepath.Join(c.Home, configName+"."+configType) + configFile := filepath.Join(c.homeDir, configName+"."+configType) if !util.PathExists(configFile) { if _, err := os.Create(configFile); err != nil { return err @@ -142,33 +156,69 @@ func (c *Config) Write() error { return viper.WriteConfig() } -func (c *Config) CertificatePath(app vespa.ApplicationID) (string, error) { - if override, ok := os.LookupEnv("VESPA_CLI_DATA_PLANE_CERT_FILE"); ok { +func (c *Config) targetType() (string, error) { + targetType, ok := c.get(targetFlag) + if !ok { + return "", fmt.Errorf("target is unset") + } + return targetType, nil +} + +func (c *Config) application() (vespa.ApplicationID, error) { + app, ok := c.get(applicationFlag) + if !ok { + return vespa.ApplicationID{}, errHint(fmt.Errorf("no application specified"), "Try the --"+applicationFlag+" flag") + } + application, err := vespa.ApplicationFromString(app) + if err != nil { + return vespa.ApplicationID{}, errHint(err, "application format is <tenant>.<app>.<instance>") + } + return application, nil +} + +func (c *Config) deploymentIn(zoneName string, system vespa.System) (vespa.Deployment, error) { + zone := system.DefaultZone + var err error + if zoneName != "" { + zone, err = vespa.ZoneFromString(zoneName) + if err != nil { + return vespa.Deployment{}, err + } + } + app, err := c.application() + if err != nil { + return vespa.Deployment{}, err + } + return vespa.Deployment{System: system, Application: app, Zone: zone}, nil +} + +func (c *Config) certificatePath(app vespa.ApplicationID) (string, error) { + if override, ok := c.environment["VESPA_CLI_DATA_PLANE_CERT_FILE"]; ok { return override, nil } return c.applicationFilePath(app, "data-plane-public-cert.pem") } -func (c *Config) PrivateKeyPath(app vespa.ApplicationID) (string, error) { - if override, ok := os.LookupEnv("VESPA_CLI_DATA_PLANE_KEY_FILE"); ok { +func (c *Config) privateKeyPath(app vespa.ApplicationID) (string, error) { + if override, ok := c.environment["VESPA_CLI_DATA_PLANE_KEY_FILE"]; ok { return override, nil } return c.applicationFilePath(app, "data-plane-private-key.pem") } -func (c *Config) X509KeyPair(app vespa.ApplicationID) (KeyPair, error) { - cert, certOk := os.LookupEnv("VESPA_CLI_DATA_PLANE_CERT") - key, keyOk := os.LookupEnv("VESPA_CLI_DATA_PLANE_KEY") +func (c *Config) x509KeyPair(app vespa.ApplicationID) (KeyPair, error) { + cert, certOk := c.environment["VESPA_CLI_DATA_PLANE_CERT"] + key, keyOk := c.environment["VESPA_CLI_DATA_PLANE_KEY"] if certOk && keyOk { // Use key pair from environment kp, err := tls.X509KeyPair([]byte(cert), []byte(key)) return KeyPair{KeyPair: kp}, err } - privateKeyFile, err := c.PrivateKeyPath(app) + privateKeyFile, err := c.privateKeyPath(app) if err != nil { return KeyPair{}, err } - certificateFile, err := c.CertificatePath(app) + certificateFile, err := c.certificatePath(app) if err != nil { return KeyPair{}, err } @@ -183,45 +233,45 @@ func (c *Config) X509KeyPair(app vespa.ApplicationID) (KeyPair, error) { }, nil } -func (c *Config) APIKeyPath(tenantName string) string { - if override, ok := c.Get(apiKeyFileFlag); ok { +func (c *Config) apiKeyPath(tenantName string) string { + if override, ok := c.get(apiKeyFileFlag); ok { return override } - return filepath.Join(c.Home, tenantName+".api-key.pem") + return filepath.Join(c.homeDir, tenantName+".api-key.pem") +} + +func (c *Config) authConfigPath() string { + return filepath.Join(c.homeDir, "auth.json") } -func (c *Config) ReadAPIKey(tenantName string) ([]byte, error) { - if override, ok := c.Get(apiKeyFlag); ok { +func (c *Config) readAPIKey(tenantName string) ([]byte, error) { + if override, ok := c.get(apiKeyFlag); ok { return []byte(override), nil } - return ioutil.ReadFile(c.APIKeyPath(tenantName)) + return ioutil.ReadFile(c.apiKeyPath(tenantName)) } -// UseAPIKey checks if api key should be used be checking if api-key or api-key-file has been set. -func (c *Config) UseAPIKey(system vespa.System, tenantName string) bool { - if _, ok := c.Get(apiKeyFlag); ok { +// useAPIKey returns true if an API key should be used when authenticating with system. +func (c *Config) useAPIKey(cli *CLI, system vespa.System, tenantName string) bool { + if _, ok := c.get(apiKeyFlag); ok { return true } - if _, ok := c.Get(apiKeyFileFlag); ok { + if _, ok := c.get(apiKeyFileFlag); ok { return true } // If no Auth0 token is created, fall back to tenant api key, but warn that this functionality is deprecated // TODO: Remove this when users have had time to migrate over to Auth0 device flow authentication - if !isCI() { - a, err := auth0.GetAuth0(c.AuthConfigPath(), system.Name, system.URL) + if !cli.isCI() { + a, err := auth0.GetAuth0(c.authConfigPath(), system.Name, system.URL) if err != nil || !a.HasSystem() { - printWarning("Use of API key is deprecated", "Authenticate with Auth0 instead: 'vespa auth login'") - return util.PathExists(c.APIKeyPath(tenantName)) + cli.printWarning("Use of API key is deprecated", "Authenticate with Auth0 instead: 'vespa auth login'") + return util.PathExists(c.apiKeyPath(tenantName)) } } return false } -func (c *Config) AuthConfigPath() string { - return filepath.Join(c.Home, "auth.json") -} - -func (c *Config) ReadSessionID(app vespa.ApplicationID) (int64, error) { +func (c *Config) readSessionID(app vespa.ApplicationID) (int64, error) { sessionPath, err := c.applicationFilePath(app, "session_id") if err != nil { return 0, err @@ -233,7 +283,7 @@ func (c *Config) ReadSessionID(app vespa.ApplicationID) (int64, error) { return strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64) } -func (c *Config) WriteSessionID(app vespa.ApplicationID, sessionID int64) error { +func (c *Config) writeSessionID(app vespa.ApplicationID, sessionID int64) error { sessionPath, err := c.applicationFilePath(app, "session_id") if err != nil { return err @@ -242,7 +292,7 @@ func (c *Config) WriteSessionID(app vespa.ApplicationID, sessionID int64) error } func (c *Config) applicationFilePath(app vespa.ApplicationID, name string) (string, error) { - appDir := filepath.Join(c.Home, app.String()) + appDir := filepath.Join(c.homeDir, app.String()) if c.createDirs { if err := os.MkdirAll(appDir, 0700); err != nil { return "", err @@ -254,14 +304,10 @@ func (c *Config) applicationFilePath(app vespa.ApplicationID, name string) (stri func (c *Config) load() error { viper.SetConfigName(configName) viper.SetConfigType(configType) - viper.AddConfigPath(c.Home) - viper.AutomaticEnv() - for option, command := range flagToConfigBindings { + viper.AddConfigPath(c.homeDir) + for option, command := range c.bindings.flag { viper.BindPFlag(option, command.PersistentFlags().Lookup(option)) } - for option, env := range envToConfigBindings { - viper.BindEnv(option, env) - } err := viper.ReadInConfig() if _, ok := err.(viper.ConfigFileNotFoundError); ok { return nil @@ -269,7 +315,12 @@ func (c *Config) load() error { return err } -func (c *Config) Get(option string) (string, bool) { +func (c *Config) get(option string) (string, bool) { + if envVar, ok := c.bindings.environment[option]; ok { + if value, ok := c.environment[envVar]; ok { + return value, true + } + } value := viper.GetString(option) if value == "" { return "", false @@ -277,7 +328,7 @@ func (c *Config) Get(option string) (string, bool) { return value, true } -func (c *Config) Set(option, value string) error { +func (c *Config) set(option, value string) error { switch option { case targetFlag: switch value { @@ -296,7 +347,7 @@ func (c *Config) Set(option, value string) error { viper.Set(option, value) return nil case waitFlag: - if _, err := strconv.ParseUint(value, 10, 32); err != nil { + if n, err := strconv.Atoi(value); err != nil || n < 0 { return fmt.Errorf("%s option must be an integer >= 0, got %q", option, value) } viper.Set(option, value) @@ -320,8 +371,8 @@ func (c *Config) Set(option, value string) error { return fmt.Errorf("invalid option or value: %q: %q", option, value) } -func printOption(cfg *Config, option string) { - value, ok := cfg.Get(option) +func (c *Config) printOption(option string) { + value, ok := c.get(option) if !ok { faintColor := color.New(color.FgWhite, color.Faint) value = faintColor.Sprint("<unset>") @@ -331,10 +382,32 @@ func printOption(cfg *Config, option string) { log.Printf("%s = %s", option, value) } -func bindFlagToConfig(option string, command *cobra.Command) { - flagToConfigBindings[option] = command +func vespaCliHome(env map[string]string) (string, error) { + home := env["VESPA_CLI_HOME"] + if home == "" { + userHome, err := os.UserHomeDir() + if err != nil { + return "", err + } + home = filepath.Join(userHome, ".vespa") + } + if err := os.MkdirAll(home, 0700); err != nil { + return "", err + } + return home, nil } -func bindEnvToConfig(option string, env string) { - envToConfigBindings[option] = env +func vespaCliCacheDir(env map[string]string) (string, error) { + cacheDir := env["VESPA_CLI_CACHE_DIR"] + if cacheDir == "" { + userCacheDir, err := os.UserCacheDir() + if err != nil { + return "", err + } + cacheDir = filepath.Join(userCacheDir, "vespa") + } + if err := os.MkdirAll(cacheDir, 0755); err != nil { + return "", err + } + return cacheDir, nil } diff --git a/client/go/cmd/config_test.go b/client/go/cmd/config_test.go index 1ca51652340..1329b356606 100644 --- a/client/go/cmd/config_test.go +++ b/client/go/cmd/config_test.go @@ -13,75 +13,64 @@ import ( ) func TestConfig(t *testing.T) { - homeDir := filepath.Join(t.TempDir(), ".vespa") - assertConfigCommandErr(t, "Error: invalid option or value: \"foo\": \"bar\"\n", homeDir, "config", "set", "foo", "bar") - assertConfigCommand(t, "foo = <unset>\n", homeDir, "config", "get", "foo") - assertConfigCommand(t, "target = local\n", homeDir, "config", "get", "target") - assertConfigCommand(t, "", homeDir, "config", "set", "target", "hosted") - assertConfigCommand(t, "target = hosted\n", homeDir, "config", "get", "target") - assertConfigCommand(t, "", homeDir, "config", "set", "target", "cloud") - assertConfigCommand(t, "target = cloud\n", homeDir, "config", "get", "target") - assertConfigCommand(t, "", homeDir, "config", "set", "target", "http://127.0.0.1:8080") - assertConfigCommand(t, "", homeDir, "config", "set", "target", "https://127.0.0.1") - assertConfigCommand(t, "target = https://127.0.0.1\n", homeDir, "config", "get", "target") - assertEnvConfigCommand(t, "api-key-file = /tmp/private.key\n", homeDir, map[string]string{"VESPA_CLI_API_KEY_FILE": "/tmp/private.key"}, "config", "get", "api-key-file") - assertConfigCommand(t, "", homeDir, "config", "set", "api-key-file", "/tmp/private.key") - assertConfigCommand(t, "api-key-file = /tmp/private.key\n", homeDir, "config", "get", "api-key-file") - - assertConfigCommandErr(t, "Error: invalid application: \"foo\"\n", homeDir, "config", "set", "application", "foo") - assertConfigCommand(t, "application = <unset>\n", homeDir, "config", "get", "application") - assertConfigCommand(t, "", homeDir, "config", "set", "application", "t1.a1.i1") - assertConfigCommand(t, "application = t1.a1.i1\n", homeDir, "config", "get", "application") - - assertConfigCommand(t, "api-key-file = /tmp/private.key\napplication = t1.a1.i1\ncolor = auto\nquiet = false\ntarget = https://127.0.0.1\nwait = 0\n", homeDir, "config", "get") - - assertConfigCommand(t, "", homeDir, "config", "set", "wait", "60") - assertConfigCommandErr(t, "Error: wait option must be an integer >= 0, got \"foo\"\n", homeDir, "config", "set", "wait", "foo") - assertConfigCommand(t, "wait = 60\n", homeDir, "config", "get", "wait") - - assertConfigCommand(t, "", homeDir, "config", "set", "quiet", "true") - assertConfigCommand(t, "", homeDir, "config", "set", "quiet", "false") + assertConfigCommandErr(t, "Error: invalid option or value: \"foo\": \"bar\"\n", "config", "set", "foo", "bar") + assertConfigCommand(t, "foo = <unset>\n", "config", "get", "foo") + assertConfigCommand(t, "target = local\n", "config", "get", "target") + assertConfigCommand(t, "", "config", "set", "target", "hosted") + assertConfigCommand(t, "target = hosted\n", "config", "get", "target") + assertConfigCommand(t, "", "config", "set", "target", "cloud") + assertConfigCommand(t, "target = cloud\n", "config", "get", "target") + assertConfigCommand(t, "", "config", "set", "target", "http://127.0.0.1:8080") + assertConfigCommand(t, "", "config", "set", "target", "https://127.0.0.1") + assertConfigCommand(t, "target = https://127.0.0.1\n", "config", "get", "target") + assertEnvConfigCommand(t, "api-key-file = /tmp/private.key\n", []string{"VESPA_CLI_API_KEY_FILE=/tmp/private.key"}, "config", "get", "api-key-file") + assertConfigCommand(t, "", "config", "set", "api-key-file", "/tmp/private.key") + assertConfigCommand(t, "api-key-file = /tmp/private.key\n", "config", "get", "api-key-file") + + assertConfigCommandErr(t, "Error: invalid application: \"foo\"\n", "config", "set", "application", "foo") + assertConfigCommand(t, "application = <unset>\n", "config", "get", "application") + assertConfigCommand(t, "", "config", "set", "application", "t1.a1.i1") + assertConfigCommand(t, "application = t1.a1.i1\n", "config", "get", "application") + + assertConfigCommand(t, "api-key-file = /tmp/private.key\napplication = t1.a1.i1\ncolor = auto\nquiet = false\ntarget = https://127.0.0.1\nwait = 0\n", "config", "get") + + assertConfigCommand(t, "", "config", "set", "wait", "60") + assertConfigCommandErr(t, "Error: wait option must be an integer >= 0, got \"foo\"\n", "config", "set", "wait", "foo") + assertConfigCommand(t, "wait = 60\n", "config", "get", "wait") + + assertConfigCommand(t, "", "config", "set", "quiet", "true") + assertConfigCommand(t, "", "config", "set", "quiet", "false") } -func assertConfigCommand(t *testing.T, expected, homeDir string, args ...string) { - assertEnvConfigCommand(t, expected, homeDir, nil, args...) +func assertConfigCommand(t *testing.T, expected string, args ...string) { + assertEnvConfigCommand(t, expected, nil, args...) } -func assertEnvConfigCommand(t *testing.T, expected, homeDir string, env map[string]string, args ...string) { - out, _ := execute(command{homeDir: homeDir, env: env, args: args}, t, nil) - assert.Equal(t, expected, out) +func assertEnvConfigCommand(t *testing.T, expected string, env []string, args ...string) { + cli, stdout, _ := newTestCLI(t, env...) + err := cli.Run(args...) + assert.Nil(t, err) + assert.Equal(t, expected, stdout.String()) } -func assertConfigCommandErr(t *testing.T, expected, homeDir string, args ...string) { - _, outErr := execute(command{homeDir: homeDir, args: args}, t, nil) - assert.Equal(t, expected, outErr) -} - -func withEnv(key, value string, fn func()) { - orig, ok := os.LookupEnv(key) - os.Setenv(key, value) - fn() - if ok { - os.Setenv(key, orig) - } else { - os.Unsetenv(key) - } +func assertConfigCommandErr(t *testing.T, expected string, args ...string) { + cli, _, stderr := newTestCLI(t) + err := cli.Run(args...) + assert.NotNil(t, err) + assert.Equal(t, expected, stderr.String()) } func TestUseAPIKey(t *testing.T) { - homeDir := t.TempDir() - c := Config{Home: homeDir} + cli, _, _ := newTestCLI(t) - assert.False(t, c.UseAPIKey(vespa.PublicSystem, "t1")) + assert.False(t, cli.config.useAPIKey(cli, vespa.PublicSystem, "t1")) - c.Set(apiKeyFileFlag, "/tmp/foo") - assert.True(t, c.UseAPIKey(vespa.PublicSystem, "t1")) - c.Set(apiKeyFileFlag, "") + cli.config.set(apiKeyFileFlag, "/tmp/foo") + assert.True(t, cli.config.useAPIKey(cli, vespa.PublicSystem, "t1")) + cli.config.set(apiKeyFileFlag, "") - withEnv("VESPA_CLI_API_KEY", "...", func() { - require.Nil(t, c.load()) - assert.True(t, c.UseAPIKey(vespa.PublicSystem, "t1")) - }) + cli, _, _ = newTestCLI(t, "VESPA_CLI_API_KEY=foo") + assert.True(t, cli.config.useAPIKey(cli, vespa.PublicSystem, "t1")) // Test deprecated functionality authContent := ` @@ -100,18 +89,10 @@ func TestUseAPIKey(t *testing.T) { } } }` - withEnv("VESPA_CLI_CLOUD_SYSTEM", "public", func() { - ci, ok := os.LookupEnv("CI") - if ok { - os.Unsetenv("CI") // Test depends on unset variable - } - _, err := os.Create(filepath.Join(homeDir, "t2.api-key.pem")) - require.Nil(t, err) - assert.True(t, c.UseAPIKey(vespa.PublicSystem, "t2")) - require.Nil(t, ioutil.WriteFile(filepath.Join(homeDir, "auth.json"), []byte(authContent), 0600)) - assert.False(t, c.UseAPIKey(vespa.PublicSystem, "t2")) - if ok { - os.Setenv("CI", ci) - } - }) + cli, _, _ = newTestCLI(t, "VESPA_CLI_CLOUD_SYSTEM=public") + _, err := os.Create(filepath.Join(cli.config.homeDir, "t2.api-key.pem")) + require.Nil(t, err) + assert.True(t, cli.config.useAPIKey(cli, vespa.PublicSystem, "t2")) + require.Nil(t, ioutil.WriteFile(filepath.Join(cli.config.homeDir, "auth.json"), []byte(authContent), 0600)) + assert.False(t, cli.config.useAPIKey(cli, vespa.PublicSystem, "t2")) } diff --git a/client/go/cmd/curl.go b/client/go/cmd/curl.go index 289a65465bd..e06942196a6 100644 --- a/client/go/cmd/curl.go +++ b/client/go/cmd/curl.go @@ -8,81 +8,77 @@ import ( "strings" "github.com/spf13/cobra" - "github.com/vespa-engine/vespa/client/go/auth0" + "github.com/vespa-engine/vespa/client/go/auth/auth0" "github.com/vespa-engine/vespa/client/go/curl" "github.com/vespa-engine/vespa/client/go/vespa" ) -var curlDryRun bool -var curlService string - -func init() { - rootCmd.AddCommand(curlCmd) - curlCmd.Flags().BoolVarP(&curlDryRun, "dry-run", "n", false, "Print the curl command that would be executed") - curlCmd.Flags().StringVarP(&curlService, "service", "s", "query", "Which service to query. Must be \"deploy\", \"document\" or \"query\"") -} - -var curlCmd = &cobra.Command{ - Use: "curl [curl-options] path", - Short: "Access Vespa directly using curl", - Long: `Access Vespa directly using curl. +func newCurlCmd(cli *CLI) *cobra.Command { + var ( + dryRun bool + curlService string + ) + cmd := &cobra.Command{ + Use: "curl [curl-options] path", + Short: "Access Vespa directly using curl", + Long: `Access Vespa directly using curl. Execute curl with the appropriate URL, certificate and private key for your application. For a more high-level interface to query and feeding, see the 'query' and 'document' commands. `, - Example: `$ vespa curl /ApplicationStatus + Example: `$ vespa curl /ApplicationStatus $ vespa curl -- -X POST -H "Content-Type:application/json" --data-binary @src/test/resources/A-Head-Full-of-Dreams.json /document/v1/namespace/music/docid/1 $ vespa curl -- -v --data-urlencode "yql=select * from music where album contains 'head';" /search/\?hits=5`, - DisableAutoGenTag: true, - SilenceUsage: true, - Args: cobra.MinimumNArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - cfg, err := LoadConfig() - if err != nil { - return err - } - target, err := getTarget() - if err != nil { - return err - } - service, err := target.Service(curlService, 0, 0, "") - if err != nil { - return err - } - url := joinURL(service.BaseURL, args[len(args)-1]) - rawArgs := args[:len(args)-1] - c, err := curl.RawArgs(url, rawArgs...) - if err != nil { - return err - } - switch curlService { - case vespa.DeployService: - if target.Type() == vespa.TargetCloud { - if err := addCloudAuth0Authentication(target.Deployment().System, cfg, c); err != nil { - return err + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + target, err := cli.target(targetOptions{}) + if err != nil { + return err + } + service, err := target.Service(curlService, 0, 0, "") + if err != nil { + return err + } + url := joinURL(service.BaseURL, args[len(args)-1]) + rawArgs := args[:len(args)-1] + c, err := curl.RawArgs(url, rawArgs...) + if err != nil { + return err + } + switch curlService { + case vespa.DeployService: + if target.Type() == vespa.TargetCloud { + if err := addCloudAuth0Authentication(target.Deployment().System, cli.config, c); err != nil { + return err + } } + case vespa.DocumentService, vespa.QueryService: + c.PrivateKey = service.TLSOptions.PrivateKeyFile + c.Certificate = service.TLSOptions.CertificateFile + default: + return fmt.Errorf("service not found: %s", curlService) } - case vespa.DocumentService, vespa.QueryService: - c.PrivateKey = service.TLSOptions.PrivateKeyFile - c.Certificate = service.TLSOptions.CertificateFile - default: - return fmt.Errorf("service not found: %s", curlService) - } - if curlDryRun { - log.Print(c.String()) - } else { - if err := c.Run(os.Stdout, os.Stderr); err != nil { - return fmt.Errorf("failed to execute curl: %w", err) + if dryRun { + log.Print(c.String()) + } else { + if err := c.Run(os.Stdout, os.Stderr); err != nil { + return fmt.Errorf("failed to execute curl: %w", err) + } } - } - return nil - }, + return nil + }, + } + cmd.Flags().BoolVarP(&dryRun, "dry-run", "n", false, "Print the curl command that would be executed") + cmd.Flags().StringVarP(&curlService, "service", "s", "query", "Which service to query. Must be \"deploy\", \"document\" or \"query\"") + return cmd } func addCloudAuth0Authentication(system vespa.System, cfg *Config, c *curl.Command) error { - a, err := auth0.GetAuth0(cfg.AuthConfigPath(), system.Name, system.URL) + a, err := auth0.GetAuth0(cfg.authConfigPath(), system.Name, system.URL) if err != nil { return err } diff --git a/client/go/cmd/curl_test.go b/client/go/cmd/curl_test.go index 50b837e0d85..520cf41e308 100644 --- a/client/go/cmd/curl_test.go +++ b/client/go/cmd/curl_test.go @@ -3,38 +3,34 @@ package cmd import ( "fmt" - "os" "path/filepath" "testing" "github.com/stretchr/testify/assert" - "github.com/vespa-engine/vespa/client/go/mock" ) func TestCurl(t *testing.T) { - homeDir := filepath.Join(t.TempDir(), ".vespa") - httpClient := &mock.HTTPClient{} - _, outErr := execute(command{args: []string{"config", "set", "application", "t1.a1.i1"}, homeDir: homeDir}, t, nil) - assert.Equal(t, "", outErr) - _, outErr = execute(command{args: []string{"config", "set", "target", "cloud"}, homeDir: homeDir}, t, nil) - assert.Equal(t, "", outErr) - _, outErr = execute(command{args: []string{"auth", "api-key"}, homeDir: homeDir}, t, nil) - assert.Equal(t, "", outErr) - _, outErr = execute(command{args: []string{"auth", "cert", "--no-add"}, homeDir: homeDir}, t, nil) - assert.Equal(t, "", outErr) + cli, stdout, _ := newTestCLI(t) + cli.Environment["VESPA_CLI_ENDPOINTS"] = "{\"endpoints\":[{\"cluster\":\"container\",\"url\":\"http://127.0.0.1:8080\"}]}" + assert.Nil(t, cli.Run("config", "set", "application", "t1.a1.i1")) + assert.Nil(t, cli.Run("config", "set", "target", "cloud")) + assert.Nil(t, cli.Run("auth", "api-key")) + assert.Nil(t, cli.Run("auth", "cert", "--no-add")) - os.Setenv("VESPA_CLI_ENDPOINTS", "{\"endpoints\":[{\"cluster\":\"container\",\"url\":\"http://127.0.0.1:8080\"}]}") - out, _ := execute(command{homeDir: homeDir, args: []string{"curl", "-n", "--", "-v", "--data-urlencode", "arg=with space", "/search"}}, t, httpClient) + stdout.Reset() + err := cli.Run("curl", "-n", "--", "-v", "--data-urlencode", "arg=with space", "/search") + assert.Nil(t, err) expected := fmt.Sprintf("curl --key %s --cert %s -v --data-urlencode 'arg=with space' http://127.0.0.1:8080/search\n", - filepath.Join(homeDir, "t1.a1.i1", "data-plane-private-key.pem"), - filepath.Join(homeDir, "t1.a1.i1", "data-plane-public-cert.pem")) - assert.Equal(t, expected, out) + filepath.Join(cli.config.homeDir, "t1.a1.i1", "data-plane-private-key.pem"), + filepath.Join(cli.config.homeDir, "t1.a1.i1", "data-plane-public-cert.pem")) + assert.Equal(t, expected, stdout.String()) - _, outErr = execute(command{args: []string{"config", "set", "target", "local"}, homeDir: homeDir}, t, nil) - assert.Equal(t, "", outErr) - out, outErr = execute(command{homeDir: homeDir, args: []string{"curl", "-a", "t1.a1.i1", "-s", "deploy", "-n", "/application/v4/tenant/foo"}}, t, httpClient) - assert.Equal(t, "", outErr) + assert.Nil(t, cli.Run("config", "set", "target", "local")) + + stdout.Reset() + err = cli.Run("curl", "-a", "t1.a1.i1", "-s", "deploy", "-n", "/application/v4/tenant/foo") + assert.Nil(t, err) expected = "curl http://127.0.0.1:19071/application/v4/tenant/foo\n" - assert.Equal(t, expected, out) + assert.Equal(t, expected, stdout.String()) } diff --git a/client/go/cmd/deploy.go b/client/go/cmd/deploy.go index 396f42fae67..cf2176b0c69 100644 --- a/client/go/cmd/deploy.go +++ b/client/go/cmd/deploy.go @@ -6,6 +6,7 @@ package cmd import ( "fmt" + "io" "log" "strconv" @@ -15,28 +16,15 @@ import ( "github.com/vespa-engine/vespa/client/go/vespa" ) -const ( - zoneFlag = "zone" - logLevelFlag = "log-level" -) - -var ( - zoneArg string - logLevelArg string -) - -func init() { - rootCmd.AddCommand(deployCmd) - rootCmd.AddCommand(prepareCmd) - rootCmd.AddCommand(activateCmd) - deployCmd.PersistentFlags().StringVarP(&zoneArg, zoneFlag, "z", "", "The zone to use for deployment. This defaults to a dev zone") - deployCmd.PersistentFlags().StringVarP(&logLevelArg, logLevelFlag, "l", "error", `Log level for Vespa logs. Must be "error", "warning", "info" or "debug"`) -} - -var deployCmd = &cobra.Command{ - Use: "deploy [application-directory]", - Short: "Deploy (prepare and activate) an application package", - Long: `Deploy (prepare and activate) an application package. +func newDeployCmd(cli *CLI) *cobra.Command { + var ( + zoneArg string + logLevelArg string + ) + cmd := &cobra.Command{ + Use: "deploy [application-directory]", + Short: "Deploy (prepare and activate) an application package", + Long: `Deploy (prepare and activate) an application package. When this returns successfully the application package has been validated and activated on config servers. The process of applying it on individual nodes @@ -47,141 +35,131 @@ If application directory is not specified, it defaults to working directory. When deploying to Vespa Cloud the system can be overridden by setting the environment variable VESPA_CLI_CLOUD_SYSTEM. This is intended for internal use only.`, - Example: `$ vespa deploy . + Example: `$ vespa deploy . $ vespa deploy -t cloud $ vespa deploy -t cloud -z dev.aws-us-east-1c # -z can be omitted here as this zone is the default $ vespa deploy -t cloud -z perf.aws-us-east-1c`, - Args: cobra.MaximumNArgs(1), - DisableAutoGenTag: true, - SilenceUsage: true, - RunE: func(cmd *cobra.Command, args []string) error { - pkg, err := vespa.FindApplicationPackage(applicationSource(args), true) - if err != nil { - return err - } - cfg, err := LoadConfig() - if err != nil { - return err - } - target, err := getTarget() - if err != nil { - return err - } - opts, err := getDeploymentOptions(cfg, pkg, target) - if err != nil { - return err - } - - var result vespa.PrepareResult - err = util.Spinner(stderr, "Uploading application package ...", func() error { - result, err = vespa.Deploy(opts) - return err - }) - if err != nil { - return err - } - - log.Println() - if opts.IsCloud() { - printSuccess("Triggered deployment of ", color.CyanString(pkg.Path), " with run ID ", color.CyanString(strconv.FormatInt(result.ID, 10))) - } else { - printSuccess("Deployed ", color.CyanString(pkg.Path)) - printPrepareLog(result) - } - if opts.IsCloud() { - log.Printf("\nUse %s for deployment status, or follow this deployment at", color.CyanString("vespa status")) - log.Print(color.CyanString(fmt.Sprintf("%s/tenant/%s/application/%s/dev/instance/%s/job/%s-%s/run/%d", - opts.Target.Deployment().System.ConsoleURL, - opts.Target.Deployment().Application.Tenant, opts.Target.Deployment().Application.Application, opts.Target.Deployment().Application.Instance, - opts.Target.Deployment().Zone.Environment, opts.Target.Deployment().Zone.Region, - result.ID))) - } - return waitForQueryService(result.ID) - }, + Args: cobra.MaximumNArgs(1), + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + pkg, err := vespa.FindApplicationPackage(applicationSource(args), true) + if err != nil { + return err + } + target, err := cli.target(targetOptions{zone: zoneArg, logLevel: logLevelArg}) + if err != nil { + return err + } + opts := cli.createDeploymentOptions(pkg, target) + + var result vespa.PrepareResult + err = util.Spinner(cli.Stderr, "Uploading application package ...", func() error { + result, err = vespa.Deploy(opts) + return err + }) + if err != nil { + return err + } + + log.Println() + if opts.IsCloud() { + cli.printSuccess("Triggered deployment of ", color.CyanString(pkg.Path), " with run ID ", color.CyanString(strconv.FormatInt(result.ID, 10))) + } else { + cli.printSuccess("Deployed ", color.CyanString(pkg.Path)) + printPrepareLog(cli.Stderr, result) + } + if opts.IsCloud() { + log.Printf("\nUse %s for deployment status, or follow this deployment at", color.CyanString("vespa status")) + log.Print(color.CyanString(fmt.Sprintf("%s/tenant/%s/application/%s/dev/instance/%s/job/%s-%s/run/%d", + opts.Target.Deployment().System.ConsoleURL, + opts.Target.Deployment().Application.Tenant, opts.Target.Deployment().Application.Application, opts.Target.Deployment().Application.Instance, + opts.Target.Deployment().Zone.Environment, opts.Target.Deployment().Zone.Region, + result.ID))) + } + return waitForQueryService(cli, result.ID) + }, + } + cmd.PersistentFlags().StringVarP(&zoneArg, "zone", "z", "", "The zone to use for deployment. This defaults to a dev zone") + cmd.PersistentFlags().StringVarP(&logLevelArg, "log-level", "l", "error", `Log level for Vespa logs. Must be "error", "warning", "info" or "debug"`) + return cmd } -var prepareCmd = &cobra.Command{ - Use: "prepare application-directory", - Short: "Prepare an application package for activation", - Args: cobra.MaximumNArgs(1), - DisableAutoGenTag: true, - SilenceUsage: true, - RunE: func(cmd *cobra.Command, args []string) error { - pkg, err := vespa.FindApplicationPackage(applicationSource(args), true) - if err != nil { - return fmt.Errorf("could not find application package: %w", err) - } - cfg, err := LoadConfig() - if err != nil { - return err - } - target, err := getTarget() - if err != nil { - return err - } - var result vespa.PrepareResult - err = util.Spinner(stderr, "Uploading application package ...", func() error { - result, err = vespa.Prepare(vespa.DeploymentOptions{ - ApplicationPackage: pkg, - Target: target, +func newPrepareCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "prepare application-directory", + Short: "Prepare an application package for activation", + Args: cobra.MaximumNArgs(1), + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + pkg, err := vespa.FindApplicationPackage(applicationSource(args), true) + if err != nil { + return fmt.Errorf("could not find application package: %w", err) + } + target, err := cli.target(targetOptions{}) + if err != nil { + return err + } + opts := cli.createDeploymentOptions(pkg, target) + var result vespa.PrepareResult + err = util.Spinner(cli.Stderr, "Uploading application package ...", func() error { + result, err = vespa.Prepare(opts) + return err }) - return err - }) - if err != nil { - return err - } - if err := cfg.WriteSessionID(vespa.DefaultApplication, result.ID); err != nil { - return fmt.Errorf("could not write session id: %w", err) - } - printSuccess("Prepared ", color.CyanString(pkg.Path), " with session ", result.ID) - return nil - }, + if err != nil { + return err + } + if err := cli.config.writeSessionID(vespa.DefaultApplication, result.ID); err != nil { + return fmt.Errorf("could not write session id: %w", err) + } + cli.printSuccess("Prepared ", color.CyanString(pkg.Path), " with session ", result.ID) + return nil + }, + } } -var activateCmd = &cobra.Command{ - Use: "activate", - Short: "Activate (deploy) a previously prepared application package", - Args: cobra.MaximumNArgs(1), - DisableAutoGenTag: true, - SilenceUsage: true, - RunE: func(cmd *cobra.Command, args []string) error { - pkg, err := vespa.FindApplicationPackage(applicationSource(args), true) - if err != nil { - return fmt.Errorf("could not find application package: %w", err) - } - cfg, err := LoadConfig() - if err != nil { - return err - } - sessionID, err := cfg.ReadSessionID(vespa.DefaultApplication) - if err != nil { - return fmt.Errorf("could not read session id: %w", err) - } - target, err := getTarget() - if err != nil { - return err - } - err = vespa.Activate(sessionID, vespa.DeploymentOptions{ - ApplicationPackage: pkg, - Target: target, - }) - if err != nil { - return err - } - printSuccess("Activated ", color.CyanString(pkg.Path), " with session ", sessionID) - return waitForQueryService(sessionID) - }, +func newActivateCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "activate", + Short: "Activate (deploy) a previously prepared application package", + Args: cobra.MaximumNArgs(1), + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + pkg, err := vespa.FindApplicationPackage(applicationSource(args), true) + if err != nil { + return fmt.Errorf("could not find application package: %w", err) + } + sessionID, err := cli.config.readSessionID(vespa.DefaultApplication) + if err != nil { + return fmt.Errorf("could not read session id: %w", err) + } + target, err := cli.target(targetOptions{}) + if err != nil { + return err + } + opts := cli.createDeploymentOptions(pkg, target) + err = vespa.Activate(sessionID, opts) + if err != nil { + return err + } + cli.printSuccess("Activated ", color.CyanString(pkg.Path), " with session ", sessionID) + return waitForQueryService(cli, sessionID) + }, + } } -func waitForQueryService(sessionOrRunID int64) error { - if waitSecsArg > 0 { +func waitForQueryService(cli *CLI, sessionOrRunID int64) error { + if cli.flags.waitSecs > 0 { log.Println() - return waitForService(vespa.QueryService, sessionOrRunID) + _, err := cli.service(vespa.QueryService, sessionOrRunID, "") + return err } return nil } -func printPrepareLog(result vespa.PrepareResult) { +func printPrepareLog(stderr io.Writer, result vespa.PrepareResult) { for _, entry := range result.LogLines { level := entry.Level switch level { diff --git a/client/go/cmd/deploy_test.go b/client/go/cmd/deploy_test.go index f5af3751eb8..e2d6da5399d 100644 --- a/client/go/cmd/deploy_test.go +++ b/client/go/cmd/deploy_test.go @@ -5,7 +5,6 @@ package cmd import ( - "path/filepath" "strconv" "testing" @@ -34,9 +33,12 @@ func TestDeployZipWithURLTargetArgument(t *testing.T) { arguments := []string{"deploy", "testdata/applications/withTarget/target/application.zip", "-t", "http://target:19071"} client := &mock.HTTPClient{} + cli, stdout, _ := newTestCLI(t) + cli.httpClient = client + assert.Nil(t, cli.Run(arguments...)) assert.Equal(t, "\nSuccess: Deployed "+applicationPackage+"\n", - executeCommand(t, client, arguments, []string{})) + stdout.String()) assertDeployRequestMade("http://target:19071", client, t) } @@ -61,11 +63,11 @@ func TestDeployApplicationDirectoryWithPomAndTarget(t *testing.T) { } func TestDeployApplicationDirectoryWithPomAndEmptyTarget(t *testing.T) { - client := &mock.HTTPClient{} - _, outErr := execute(command{args: []string{"deploy", "testdata/applications/withEmptyTarget"}}, t, client) + cli, _, stderr := newTestCLI(t) + assert.NotNil(t, cli.Run("deploy", "testdata/applications/withEmptyTarget")) assert.Equal(t, "Error: pom.xml exists but no target/application.zip. Run mvn package first\n", - outErr) + stderr.String()) } func TestDeployApplicationPackageErrorWithUnexpectedNonJson(t *testing.T) { @@ -105,19 +107,25 @@ func TestDeployError(t *testing.T) { } func assertDeploy(applicationPackage string, arguments []string, t *testing.T) { + cli, stdout, _ := newTestCLI(t) client := &mock.HTTPClient{} + cli.httpClient = client + assert.Nil(t, cli.Run(arguments...)) assert.Equal(t, "\nSuccess: Deployed "+applicationPackage+"\n", - executeCommand(t, client, arguments, []string{})) + stdout.String()) assertDeployRequestMade("http://127.0.0.1:19071", client, t) } func assertPrepare(applicationPackage string, arguments []string, t *testing.T) { client := &mock.HTTPClient{} client.NextResponse(200, `{"session-id":"42"}`) + cli, stdout, _ := newTestCLI(t) + cli.httpClient = client + assert.Nil(t, cli.Run(arguments...)) assert.Equal(t, "Success: Prepared "+applicationPackage+" with session 42\n", - executeCommand(t, client, arguments, []string{})) + stdout.String()) assertPackageUpload(0, "http://127.0.0.1:19071/application/v2/tenant/default/session", client, t) sessionURL := "http://127.0.0.1:19071/application/v2/tenant/default/session/42/prepared" @@ -127,15 +135,15 @@ func assertPrepare(applicationPackage string, arguments []string, t *testing.T) func assertActivate(applicationPackage string, arguments []string, t *testing.T) { client := &mock.HTTPClient{} - homeDir := t.TempDir() - cfg := Config{Home: filepath.Join(homeDir, ".vespa"), createDirs: true} - if err := cfg.WriteSessionID(vespa.DefaultApplication, 42); err != nil { + cli, stdout, _ := newTestCLI(t) + cli.httpClient = client + if err := cli.config.writeSessionID(vespa.DefaultApplication, 42); err != nil { t.Fatal(err) } - out, _ := execute(command{args: arguments, homeDir: cfg.Home}, t, client) + assert.Nil(t, cli.Run(arguments...)) assert.Equal(t, "Success: Activated "+applicationPackage+" with session 42\n", - out) + stdout.String()) url := "http://127.0.0.1:19071/application/v2/tenant/default/session/42/active" assert.Equal(t, url, client.LastRequest.URL.String()) assert.Equal(t, "PUT", client.LastRequest.Method) @@ -163,17 +171,21 @@ func assertDeployRequestMade(target string, client *mock.HTTPClient, t *testing. func assertApplicationPackageError(t *testing.T, cmd string, status int, expectedMessage string, returnBody string) { client := &mock.HTTPClient{} client.NextResponse(status, returnBody) - _, outErr := execute(command{args: []string{cmd, "testdata/applications/withTarget/target/application.zip"}}, t, client) + cli, _, stderr := newTestCLI(t) + cli.httpClient = client + assert.NotNil(t, cli.Run(cmd, "testdata/applications/withTarget/target/application.zip")) assert.Equal(t, "Error: invalid application package (Status "+strconv.Itoa(status)+")\n"+expectedMessage+"\n", - outErr) + stderr.String()) } func assertDeployServerError(t *testing.T, status int, errorMessage string) { client := &mock.HTTPClient{} client.NextResponse(status, errorMessage) - _, outErr := execute(command{args: []string{"deploy", "testdata/applications/withTarget/target/application.zip"}}, t, client) + cli, _, stderr := newTestCLI(t) + cli.httpClient = client + assert.NotNil(t, cli.Run("deploy", "testdata/applications/withTarget/target/application.zip")) assert.Equal(t, "Error: error from deploy api at 127.0.0.1:19071 (Status "+strconv.Itoa(status)+"):\n"+errorMessage+"\n", - outErr) + stderr.String()) } diff --git a/client/go/cmd/document.go b/client/go/cmd/document.go index 5e5108d117d..1c155bd6718 100644 --- a/client/go/cmd/document.go +++ b/client/go/cmd/document.go @@ -17,25 +17,20 @@ import ( "github.com/vespa-engine/vespa/client/go/vespa" ) -var ( - docPrintCurl bool - docTimeoutSecs int -) - -func init() { - rootCmd.AddCommand(documentCmd) - documentCmd.AddCommand(documentPutCmd) - documentCmd.AddCommand(documentUpdateCmd) - documentCmd.AddCommand(documentRemoveCmd) - documentCmd.AddCommand(documentGetCmd) - documentCmd.PersistentFlags().BoolVarP(&docPrintCurl, "verbose", "v", false, "Print the equivalent curl command for the document operation") - documentCmd.PersistentFlags().IntVarP(&docTimeoutSecs, "timeout", "T", 60, "Timeout for the document request in seconds") +func addDocumentFlags(cmd *cobra.Command, printCurl *bool, timeoutSecs *int) { + cmd.PersistentFlags().BoolVarP(printCurl, "verbose", "v", false, "Print the equivalent curl command for the document operation") + cmd.PersistentFlags().IntVarP(timeoutSecs, "timeout", "T", 60, "Timeout for the document request in seconds") } -var documentCmd = &cobra.Command{ - Use: "document json-file", - Short: "Issue a document operation to Vespa", - Long: `Issue a document operation to Vespa. +func newDocumentCmd(cli *CLI) *cobra.Command { + var ( + printCurl bool + timeoutSecs int + ) + cmd := &cobra.Command{ + Use: "document json-file", + Short: "Issue a document operation to Vespa", + Long: `Issue a document operation to Vespa. The operation must be on the format documented in https://docs.vespa.ai/en/reference/document-json-format.html#document-operations @@ -45,125 +40,159 @@ subsequent get or query operation. To feed with high throughput, https://docs.vespa.ai/en/vespa-feed-client.html should be used instead of this.`, - Example: `$ vespa document src/test/resources/A-Head-Full-of-Dreams.json`, - DisableAutoGenTag: true, - SilenceUsage: true, - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - service, err := documentService() - if err != nil { - return err - } - return printResult(vespa.Send(args[0], service, operationOptions()), false) - }, + Example: `$ vespa document src/test/resources/A-Head-Full-of-Dreams.json`, + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + service, err := documentService(cli) + if err != nil { + return err + } + return printResult(cli, vespa.Send(args[0], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), false) + }, + } + addDocumentFlags(cmd, &printCurl, &timeoutSecs) + return cmd } -var documentPutCmd = &cobra.Command{ - Use: "put [id] json-file", - Short: "Writes a document to Vespa", - Long: `Writes the document in the given file to Vespa. +func newDocumentPutCmd(cli *CLI) *cobra.Command { + var ( + printCurl bool + timeoutSecs int + ) + cmd := &cobra.Command{ + Use: "put [id] json-file", + Short: "Writes a document to Vespa", + Long: `Writes the document in the given file to Vespa. If the document already exists, all its values will be replaced by this document. If the document id is specified both as an argument and in the file the argument takes precedence.`, - Args: cobra.RangeArgs(1, 2), - Example: `$ vespa document put src/test/resources/A-Head-Full-of-Dreams.json + Args: cobra.RangeArgs(1, 2), + Example: `$ vespa document put src/test/resources/A-Head-Full-of-Dreams.json $ vespa document put id:mynamespace:music::a-head-full-of-dreams src/test/resources/A-Head-Full-of-Dreams.json`, - DisableAutoGenTag: true, - SilenceUsage: true, - RunE: func(cmd *cobra.Command, args []string) error { - service, err := documentService() - if err != nil { - return err - } - if len(args) == 1 { - return printResult(vespa.Put("", args[0], service, operationOptions()), false) - } else { - return printResult(vespa.Put(args[0], args[1], service, operationOptions()), false) - } - }, + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + service, err := documentService(cli) + if err != nil { + return err + } + if len(args) == 1 { + return printResult(cli, vespa.Put("", args[0], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), false) + } else { + return printResult(cli, vespa.Put(args[0], args[1], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), false) + } + }, + } + addDocumentFlags(cmd, &printCurl, &timeoutSecs) + return cmd } -var documentUpdateCmd = &cobra.Command{ - Use: "update [id] json-file", - Short: "Modifies some fields of an existing document", - Long: `Updates the values of the fields given in a json file as specified in the file. +func newDocumentUpdateCmd(cli *CLI) *cobra.Command { + var ( + printCurl bool + timeoutSecs int + ) + cmd := &cobra.Command{ + Use: "update [id] json-file", + Short: "Modifies some fields of an existing document", + Long: `Updates the values of the fields given in a json file as specified in the file. If the document id is specified both as an argument and in the file the argument takes precedence.`, - Args: cobra.RangeArgs(1, 2), - Example: `$ vespa document update src/test/resources/A-Head-Full-of-Dreams-Update.json + Args: cobra.RangeArgs(1, 2), + Example: `$ vespa document update src/test/resources/A-Head-Full-of-Dreams-Update.json $ vespa document update id:mynamespace:music::a-head-full-of-dreams src/test/resources/A-Head-Full-of-Dreams.json`, - DisableAutoGenTag: true, - SilenceUsage: true, - RunE: func(cmd *cobra.Command, args []string) error { - service, err := documentService() - if err != nil { - return err - } - if len(args) == 1 { - return printResult(vespa.Update("", args[0], service, operationOptions()), false) - } else { - return printResult(vespa.Update(args[0], args[1], service, operationOptions()), false) - } - }, + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + service, err := documentService(cli) + if err != nil { + return err + } + if len(args) == 1 { + return printResult(cli, vespa.Update("", args[0], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), false) + } else { + return printResult(cli, vespa.Update(args[0], args[1], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), false) + } + }, + } + addDocumentFlags(cmd, &printCurl, &timeoutSecs) + return cmd } -var documentRemoveCmd = &cobra.Command{ - Use: "remove id | json-file", - Short: "Removes a document from Vespa", - Long: `Removes the document specified either as a document id or given in the json file. +func newDocumentRemoveCmd(cli *CLI) *cobra.Command { + var ( + printCurl bool + timeoutSecs int + ) + cmd := &cobra.Command{ + Use: "remove id | json-file", + Short: "Removes a document from Vespa", + Long: `Removes the document specified either as a document id or given in the json file. If the document id is specified both as an argument and in the file the argument takes precedence.`, - Args: cobra.ExactArgs(1), - Example: `$ vespa document remove src/test/resources/A-Head-Full-of-Dreams-Remove.json + Args: cobra.ExactArgs(1), + Example: `$ vespa document remove src/test/resources/A-Head-Full-of-Dreams-Remove.json $ vespa document remove id:mynamespace:music::a-head-full-of-dreams`, - DisableAutoGenTag: true, - SilenceUsage: true, - RunE: func(cmd *cobra.Command, args []string) error { - service, err := documentService() - if err != nil { - return err - } - if strings.HasPrefix(args[0], "id:") { - return printResult(vespa.RemoveId(args[0], service, operationOptions()), false) - } else { - return printResult(vespa.RemoveOperation(args[0], service, operationOptions()), false) - } - }, + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + service, err := documentService(cli) + if err != nil { + return err + } + if strings.HasPrefix(args[0], "id:") { + return printResult(cli, vespa.RemoveId(args[0], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), false) + } else { + return printResult(cli, vespa.RemoveOperation(args[0], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), false) + } + }, + } + addDocumentFlags(cmd, &printCurl, &timeoutSecs) + return cmd } -var documentGetCmd = &cobra.Command{ - Use: "get id", - Short: "Gets a document", - Args: cobra.ExactArgs(1), - DisableAutoGenTag: true, - SilenceUsage: true, - Example: `$ vespa document get id:mynamespace:music::a-head-full-of-dreams`, - RunE: func(cmd *cobra.Command, args []string) error { - service, err := documentService() - if err != nil { - return err - } - return printResult(vespa.Get(args[0], service, operationOptions()), true) - }, +func newDocumentGetCmd(cli *CLI) *cobra.Command { + var ( + printCurl bool + timeoutSecs int + ) + cmd := &cobra.Command{ + Use: "get id", + Short: "Gets a document", + Args: cobra.ExactArgs(1), + DisableAutoGenTag: true, + SilenceUsage: true, + Example: `$ vespa document get id:mynamespace:music::a-head-full-of-dreams`, + RunE: func(cmd *cobra.Command, args []string) error { + service, err := documentService(cli) + if err != nil { + return err + } + return printResult(cli, vespa.Get(args[0], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), true) + }, + } + addDocumentFlags(cmd, &printCurl, &timeoutSecs) + return cmd } -func documentService() (*vespa.Service, error) { return getService(vespa.DocumentService, 0, "") } - -func operationOptions() vespa.OperationOptions { - return vespa.OperationOptions{ - CurlOutput: curlOutput(), - Timeout: time.Second * time.Duration(docTimeoutSecs), - } +func documentService(cli *CLI) (*vespa.Service, error) { + return cli.service(vespa.DocumentService, 0, "") } -func curlOutput() io.Writer { - if docPrintCurl { - return stderr +func operationOptions(stderr io.Writer, printCurl bool, timeoutSecs int) vespa.OperationOptions { + curlOutput := ioutil.Discard + if printCurl { + curlOutput = stderr + } + return vespa.OperationOptions{ + CurlOutput: curlOutput, + Timeout: time.Second * time.Duration(timeoutSecs), } - return ioutil.Discard } -func printResult(result util.OperationResult, payloadOnlyOnSuccess bool) error { - out := stdout +func printResult(cli *CLI, result util.OperationResult, payloadOnlyOnSuccess bool) error { + out := cli.Stdout if !result.Success { - out = stderr + out = cli.Stderr } if !result.Success { diff --git a/client/go/cmd/document_test.go b/client/go/cmd/document_test.go index 1d650f77d08..77b7d68d666 100644 --- a/client/go/cmd/document_test.go +++ b/client/go/cmd/document_test.go @@ -66,21 +66,19 @@ func TestDocumentRemoveWithoutIdArg(t *testing.T) { } func TestDocumentSendMissingId(t *testing.T) { - arguments := []string{"document", "put", "testdata/A-Head-Full-of-Dreams-Without-Operation.json"} - client := &mock.HTTPClient{} - _, outErr := execute(command{args: arguments}, t, client) + cli, _, stderr := newTestCLI(t) + assert.NotNil(t, cli.Run("document", "put", "testdata/A-Head-Full-of-Dreams-Without-Operation.json")) assert.Equal(t, "Error: No document id given neither as argument or as a 'put' key in the json file\n", - outErr) + stderr.String()) } func TestDocumentSendWithDisagreeingOperations(t *testing.T) { - arguments := []string{"document", "update", "testdata/A-Head-Full-of-Dreams-Put.json"} - client := &mock.HTTPClient{} - _, outErr := execute(command{args: arguments}, t, client) + cli, _, stderr := newTestCLI(t) + assert.NotNil(t, cli.Run("document", "update", "testdata/A-Head-Full-of-Dreams-Put.json")) assert.Equal(t, "Error: Wanted document operation is update but the JSON file specifies put\n", - outErr) + stderr.String()) } func TestDocumentPutDocumentError(t *testing.T) { @@ -98,14 +96,16 @@ func TestDocumentGet(t *testing.T) { func assertDocumentSend(arguments []string, expectedOperation string, expectedMethod string, expectedDocumentId string, expectedPayloadFile string, t *testing.T) { client := &mock.HTTPClient{} + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = client documentURL, err := documentServiceURL(client) if err != nil { t.Fatal(err) } expectedPath, _ := vespa.IdToURLPath(expectedDocumentId) expectedURL := documentURL + "/document/v1/" + expectedPath - out, errOut := execute(command{args: arguments}, t, client) + assert.Nil(t, cli.Run(arguments...)) verbose := false for _, a := range arguments { if a == "-v" { @@ -114,9 +114,9 @@ func assertDocumentSend(arguments []string, expectedOperation string, expectedMe } if verbose { expectedCurl := "curl -X " + expectedMethod + " -H 'Content-Type: application/json' --data-binary @" + expectedPayloadFile + " " + expectedURL + "\n" - assert.Equal(t, expectedCurl, errOut) + assert.Equal(t, expectedCurl, stderr.String()) } - assert.Equal(t, "Success: "+expectedOperation+" "+expectedDocumentId+"\n", out) + assert.Equal(t, "Success: "+expectedOperation+" "+expectedDocumentId+"\n", stdout.String()) assert.Equal(t, expectedURL, client.LastRequest.URL.String()) assert.Equal(t, "application/json", client.LastRequest.Header.Get("Content-Type")) assert.Equal(t, expectedMethod, client.LastRequest.Method) @@ -132,6 +132,9 @@ func assertDocumentGet(arguments []string, documentId string, t *testing.T) { t.Fatal(err) } client.NextResponse(200, "{\"fields\":{\"foo\":\"bar\"}}") + cli, stdout, _ := newTestCLI(t) + cli.httpClient = client + assert.Nil(t, cli.Run(arguments...)) assert.Equal(t, `{ "fields": { @@ -139,7 +142,7 @@ func assertDocumentGet(arguments []string, documentId string, t *testing.T) { } } `, - executeCommand(t, client, arguments, []string{})) + stdout.String()) expectedPath, _ := vespa.IdToURLPath(documentId) assert.Equal(t, documentURL+"/document/v1/"+expectedPath, client.LastRequest.URL.String()) assert.Equal(t, "GET", client.LastRequest.Method) @@ -148,29 +151,29 @@ func assertDocumentGet(arguments []string, documentId string, t *testing.T) { func assertDocumentError(t *testing.T, status int, errorMessage string) { client := &mock.HTTPClient{} client.NextResponse(status, errorMessage) - _, outErr := execute(command{args: []string{"document", "put", + cli, _, stderr := newTestCLI(t) + cli.httpClient = client + assert.NotNil(t, cli.Run("document", "put", "id:mynamespace:music::a-head-full-of-dreams", - "testdata/A-Head-Full-of-Dreams-Put.json"}}, t, client) + "testdata/A-Head-Full-of-Dreams-Put.json")) assert.Equal(t, "Error: Invalid document operation: Status "+strconv.Itoa(status)+"\n\n"+errorMessage+"\n", - outErr) + stderr.String()) } func assertDocumentServerError(t *testing.T, status int, errorMessage string) { client := &mock.HTTPClient{} client.NextResponse(status, errorMessage) - _, outErr := execute(command{args: []string{"document", "put", + cli, _, stderr := newTestCLI(t) + cli.httpClient = client + assert.NotNil(t, cli.Run("document", "put", "id:mynamespace:music::a-head-full-of-dreams", - "testdata/A-Head-Full-of-Dreams-Put.json"}}, t, client) + "testdata/A-Head-Full-of-Dreams-Put.json")) assert.Equal(t, "Error: Container (document API) at 127.0.0.1:8080: Status "+strconv.Itoa(status)+"\n\n"+errorMessage+"\n", - outErr) + stderr.String()) } func documentServiceURL(client *mock.HTTPClient) (string, error) { - service, err := getService("document", 0, "") - if err != nil { - return "", err - } - return service.BaseURL, nil + return "http://127.0.0.1:8080", nil } diff --git a/client/go/cmd/helpers.go b/client/go/cmd/helpers.go deleted file mode 100644 index d45fda58a2f..00000000000 --- a/client/go/cmd/helpers.go +++ /dev/null @@ -1,368 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Helpers used by multiple sub-commands. -// Author: mpolden - -package cmd - -import ( - "crypto/tls" - "crypto/x509" - "encoding/json" - "fmt" - "log" - "os" - "path/filepath" - "strconv" - "strings" - "time" - - "github.com/fatih/color" - "github.com/vespa-engine/vespa/client/go/build" - "github.com/vespa-engine/vespa/client/go/version" - "github.com/vespa-engine/vespa/client/go/vespa" -) - -func printErrHint(err error, hints ...string) { - fmt.Fprintln(stderr, color.RedString("Error:"), err) - for _, hint := range hints { - fmt.Fprintln(stderr, color.CyanString("Hint:"), hint) - } -} - -func printSuccess(msg ...interface{}) { - log.Print(color.GreenString("Success: "), fmt.Sprint(msg...)) -} - -func printWarning(msg string, hints ...string) { - fmt.Fprintln(stderr, color.YellowString("Warning:"), msg) - for _, hint := range hints { - fmt.Fprintln(stderr, color.CyanString("Hint:"), hint) - } -} - -func athenzPath(filename string) (string, error) { - userHome, err := os.UserHomeDir() - if err != nil { - return "", err - } - return filepath.Join(userHome, ".athenz", filename), nil -} - -func athenzKeyPair() (KeyPair, error) { - certFile, err := athenzPath("cert") - if err != nil { - return KeyPair{}, err - } - keyFile, err := athenzPath("key") - if err != nil { - return KeyPair{}, err - } - kp, err := tls.LoadX509KeyPair(certFile, keyFile) - if err != nil { - return KeyPair{}, err - } - cert, err := x509.ParseCertificate(kp.Certificate[0]) - if err != nil { - return KeyPair{}, err - } - now := time.Now() - expiredAt := cert.NotAfter - if expiredAt.Before(now) { - delta := now.Sub(expiredAt).Truncate(time.Second) - return KeyPair{}, errHint(fmt.Errorf("certificate %s expired at %s (%s ago)", certFile, cert.NotAfter, delta), "Try renewing certificate with 'athenz-user-cert'") - } - return KeyPair{KeyPair: kp, CertificateFile: certFile, PrivateKeyFile: keyFile}, nil -} - -func vespaCliHome() (string, error) { - home := os.Getenv("VESPA_CLI_HOME") - if home == "" { - userHome, err := os.UserHomeDir() - if err != nil { - return "", err - } - home = filepath.Join(userHome, ".vespa") - } - if err := os.MkdirAll(home, 0700); err != nil { - return "", err - } - return home, nil -} - -func vespaCliCacheDir() (string, error) { - cacheDir := os.Getenv("VESPA_CLI_CACHE_DIR") - if cacheDir == "" { - userCacheDir, err := os.UserCacheDir() - if err != nil { - return "", err - } - cacheDir = filepath.Join(userCacheDir, "vespa") - } - if err := os.MkdirAll(cacheDir, 0755); err != nil { - return "", err - } - return cacheDir, nil -} - -func deploymentFromArgs(system vespa.System) (vespa.Deployment, error) { - zone := system.DefaultZone - var err error - if zoneArg != "" { - zone, err = vespa.ZoneFromString(zoneArg) - if err != nil { - return vespa.Deployment{}, err - } - } - app, err := getApplication() - if err != nil { - return vespa.Deployment{}, err - } - return vespa.Deployment{System: system, Application: app, Zone: zone}, nil -} - -func applicationSource(args []string) string { - if len(args) > 0 { - return args[0] - } - return "." -} - -func getApplication() (vespa.ApplicationID, error) { - cfg, err := LoadConfig() - if err != nil { - return vespa.ApplicationID{}, err - } - app, ok := cfg.Get(applicationFlag) - if !ok { - return vespa.ApplicationID{}, errHint(fmt.Errorf("no application specified: %w", err), "Try the --"+applicationFlag+" flag") - } - application, err := vespa.ApplicationFromString(app) - if err != nil { - return vespa.ApplicationID{}, errHint(err, "application format is <tenant>.<app>.<instance>") - } - return application, nil -} - -func getTargetType() (string, error) { - cfg, err := LoadConfig() - if err != nil { - return "", err - } - target, ok := cfg.Get(targetFlag) - if !ok { - return "", fmt.Errorf("target is unset") - } - return target, nil -} - -func getService(service string, sessionOrRunID int64, cluster string) (*vespa.Service, error) { - t, err := getTarget() - if err != nil { - return nil, err - } - timeout := time.Duration(waitSecsArg) * time.Second - if timeout > 0 { - log.Printf("Waiting up to %s %s for %s service to become available ...", color.CyanString(strconv.Itoa(waitSecsArg)), color.CyanString("seconds"), color.CyanString(service)) - } - s, err := t.Service(service, timeout, sessionOrRunID, cluster) - if err != nil { - return nil, fmt.Errorf("service '%s' is unavailable: %w", service, err) - } - return s, nil -} - -func getEndpointsOverride() string { return os.Getenv("VESPA_CLI_ENDPOINTS") } - -func getSystem(targetType string) (vespa.System, error) { - name := os.Getenv("VESPA_CLI_CLOUD_SYSTEM") - if name != "" { - return vespa.GetSystem(name) - } - switch targetType { - case vespa.TargetHosted: - return vespa.MainSystem, nil - case vespa.TargetCloud: - return vespa.PublicSystem, nil - } - return vespa.System{}, fmt.Errorf("no default system found for %s target", targetType) -} - -func getTarget() (vespa.Target, error) { - clientVersion, err := version.Parse(build.Version) - if err != nil { - return nil, err - } - target, err := createTarget() - if err != nil { - return nil, err - } - if !isCloudCI() { // Vespa Cloud always runs an up-to-date version - if err := target.CheckVersion(clientVersion); err != nil { - printErrHint(err, "This is not a fatal error, but this version may not work as expected", "Try 'vespa version' to check for a new version") - } - } - return target, nil -} - -func createTarget() (vespa.Target, error) { - targetType, err := getTargetType() - if err != nil { - return nil, err - } - if strings.HasPrefix(targetType, "http") { - return vespa.CustomTarget(targetType), nil - } - switch targetType { - case vespa.TargetLocal: - return vespa.LocalTarget(), nil - case vespa.TargetCloud, vespa.TargetHosted: - return createCloudTarget(targetType) - } - return nil, errHint(fmt.Errorf("invalid target: %s", targetType), "Valid targets are 'local', 'cloud', 'hosted' or an URL") -} - -func createCloudTarget(targetType string) (vespa.Target, error) { - cfg, err := LoadConfig() - if err != nil { - return nil, err - } - system, err := getSystem(targetType) - if err != nil { - return nil, err - } - deployment, err := deploymentFromArgs(system) - if err != nil { - return nil, err - } - endpoints, err := getEndpointsFromEnv() - if err != nil { - return nil, err - } - var ( - apiKey []byte - authConfigPath string - apiTLSOptions vespa.TLSOptions - deploymentTLSOptions vespa.TLSOptions - ) - if targetType == vespa.TargetCloud { - if cfg.UseAPIKey(system, deployment.Application.Tenant) { - apiKey, err = cfg.ReadAPIKey(deployment.Application.Tenant) - if err != nil { - return nil, err - } - } - authConfigPath = cfg.AuthConfigPath() - kp, err := cfg.X509KeyPair(deployment.Application) - if err != nil { - return nil, errHint(err, "Deployment to cloud requires a certificate. Try 'vespa auth cert'") - } - deploymentTLSOptions = vespa.TLSOptions{ - KeyPair: kp.KeyPair, - CertificateFile: kp.CertificateFile, - PrivateKeyFile: kp.PrivateKeyFile, - } - } else if targetType == vespa.TargetHosted { - kp, err := athenzKeyPair() - if err != nil { - return nil, err - } - apiTLSOptions = vespa.TLSOptions{ - KeyPair: kp.KeyPair, - CertificateFile: kp.CertificateFile, - PrivateKeyFile: kp.PrivateKeyFile, - } - deploymentTLSOptions = apiTLSOptions - } else { - return nil, fmt.Errorf("invalid cloud target: %s", targetType) - } - apiOptions := vespa.APIOptions{ - System: system, - TLSOptions: apiTLSOptions, - APIKey: apiKey, - AuthConfigPath: authConfigPath, - } - deploymentOptions := vespa.CloudDeploymentOptions{ - Deployment: deployment, - TLSOptions: deploymentTLSOptions, - ClusterURLs: endpoints, - } - logOptions := vespa.LogOptions{ - Writer: stdout, - Level: vespa.LogLevel(logLevelArg), - } - return vespa.CloudTarget(apiOptions, deploymentOptions, logOptions) -} - -func waitForService(service string, sessionOrRunID int64) error { - s, err := getService(service, sessionOrRunID, "") - if err != nil { - return err - } - timeout := time.Duration(waitSecsArg) * time.Second - if timeout > 0 { - log.Printf("Waiting up to %s %s for service to become ready ...", color.CyanString(strconv.Itoa(waitSecsArg)), color.CyanString("seconds")) - } - status, err := s.Wait(timeout) - if status/100 == 2 { - log.Print(s.Description(), " at ", color.CyanString(s.BaseURL), " is ", color.GreenString("ready")) - } else { - if err == nil { - err = fmt.Errorf("status %d", status) - } - return fmt.Errorf("%s at %s is %s: %w", s.Description(), color.CyanString(s.BaseURL), color.RedString("not ready"), err) - } - return nil -} - -func getDeploymentOptions(cfg *Config, pkg vespa.ApplicationPackage, target vespa.Target) (vespa.DeploymentOptions, error) { - opts := vespa.DeploymentOptions{ApplicationPackage: pkg, Target: target} - if opts.IsCloud() { - if target.Type() == vespa.TargetCloud && !opts.ApplicationPackage.HasCertificate() { - hint := "Try 'vespa auth cert'" - return vespa.DeploymentOptions{}, errHint(fmt.Errorf("missing certificate in application package"), "Applications in Vespa Cloud require a certificate", hint) - } - } - opts.Timeout = time.Duration(waitSecsArg) * time.Second - return opts, nil -} - -func getEndpointsFromEnv() (map[string]string, error) { - endpointsString := getEndpointsOverride() - if endpointsString == "" { - return nil, nil - } - - var endpoints endpoints - urlsByCluster := make(map[string]string) - if err := json.Unmarshal([]byte(endpointsString), &endpoints); err != nil { - return nil, fmt.Errorf("endpoints must be valid json: %w", err) - } - if len(endpoints.Endpoints) == 0 { - return nil, fmt.Errorf("endpoints must be non-empty") - } - for _, endpoint := range endpoints.Endpoints { - urlsByCluster[endpoint.Cluster] = endpoint.URL - } - return urlsByCluster, nil -} - -// isCI returns true if running inside a continuous integration environment. -func isCI() bool { - _, ok := os.LookupEnv("CI") - return ok -} - -// isCloudCI returns true if running inside a Vespa Cloud deployment job. -func isCloudCI() bool { - _, ok := os.LookupEnv("VESPA_CLI_CLOUD_CI") - return ok -} - -type endpoints struct { - Endpoints []endpoint `json:"endpoints"` -} - -type endpoint struct { - Cluster string `json:"cluster"` - URL string `json:"url"` -} diff --git a/client/go/cmd/log.go b/client/go/cmd/log.go index d61eaecf35b..dffad8896fc 100644 --- a/client/go/cmd/log.go +++ b/client/go/cmd/log.go @@ -9,72 +9,70 @@ import ( "github.com/vespa-engine/vespa/client/go/vespa" ) -var ( - fromArg string - toArg string - levelArg string - followArg bool - dequoteArg bool -) - -func init() { - rootCmd.AddCommand(logCmd) - logCmd.Flags().StringVarP(&fromArg, "from", "F", "", "Include logs since this timestamp (RFC3339 format)") - logCmd.Flags().StringVarP(&toArg, "to", "T", "", "Include logs until this timestamp (RFC3339 format)") - logCmd.Flags().StringVarP(&levelArg, "level", "l", "debug", `The maximum log level to show. Must be "error", "warning", "info" or "debug"`) - logCmd.Flags().BoolVarP(&followArg, "follow", "f", false, "Follow logs") - logCmd.Flags().BoolVarP(&dequoteArg, "nldequote", "n", true, "Dequote LF and TAB characters in log messages") -} - -var logCmd = &cobra.Command{ - Use: "log [relative-period]", - Short: "Show the Vespa log", - Long: `Show the Vespa log. +func newLogCmd(cli *CLI) *cobra.Command { + var ( + fromArg string + toArg string + levelArg string + followArg bool + dequoteArg bool + ) + cmd := &cobra.Command{ + Use: "log [relative-period]", + Short: "Show the Vespa log", + Long: `Show the Vespa log. The logs shown can be limited to a relative or fixed period. All timestamps are shown in UTC. Logs for the past hour are shown if no arguments are given. `, - Example: `$ vespa log 1h + Example: `$ vespa log 1h $ vespa log --nldequote=false 10m $ vespa log --from 2021-08-25T15:00:00Z --to 2021-08-26T02:00:00Z $ vespa log --follow`, - DisableAutoGenTag: true, - SilenceUsage: true, - Args: cobra.MaximumNArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - target, err := getTarget() - if err != nil { - return err - } - options := vespa.LogOptions{ - Level: vespa.LogLevel(levelArg), - Follow: followArg, - Writer: stdout, - Dequote: dequoteArg, - } - if options.Follow { - if fromArg != "" || toArg != "" || len(args) > 0 { - return fmt.Errorf("cannot combine --from/--to or relative time with --follow") - } - options.From = time.Now().Add(-5 * time.Minute) - } else { - from, to, err := parsePeriod(args) + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + target, err := cli.target(targetOptions{logLevel: levelArg}) if err != nil { - return fmt.Errorf("invalid period: %w", err) + return err } - options.From = from - options.To = to - } - if err := target.PrintLog(options); err != nil { - return fmt.Errorf("could not retrieve logs: %w", err) - } - return nil - }, + options := vespa.LogOptions{ + Level: vespa.LogLevel(levelArg), + Follow: followArg, + Writer: cli.Stdout, + Dequote: dequoteArg, + } + if options.Follow { + if fromArg != "" || toArg != "" || len(args) > 0 { + return fmt.Errorf("cannot combine --from/--to or relative time with --follow") + } + options.From = time.Now().Add(-5 * time.Minute) + } else { + from, to, err := parsePeriod(fromArg, toArg, args) + if err != nil { + return fmt.Errorf("invalid period: %w", err) + } + options.From = from + options.To = to + } + if err := target.PrintLog(options); err != nil { + return fmt.Errorf("could not retrieve logs: %w", err) + } + return nil + }, + } + cmd.Flags().StringVarP(&fromArg, "from", "F", "", "Include logs since this timestamp (RFC3339 format)") + cmd.Flags().StringVarP(&toArg, "to", "T", "", "Include logs until this timestamp (RFC3339 format)") + cmd.Flags().StringVarP(&levelArg, "level", "l", "debug", `The maximum log level to show. Must be "error", "warning", "info" or "debug"`) + cmd.Flags().BoolVarP(&followArg, "follow", "f", false, "Follow logs") + cmd.Flags().BoolVarP(&dequoteArg, "nldequote", "n", true, "Dequote LF and TAB characters in log messages") + return cmd } -func parsePeriod(args []string) (time.Time, time.Time, error) { - relativePeriod := fromArg == "" || toArg == "" +func parsePeriod(from, to string, args []string) (time.Time, time.Time, error) { + relativePeriod := from == "" || to == "" if relativePeriod { period := "1h" if len(args) > 0 { @@ -93,16 +91,16 @@ func parsePeriod(args []string) (time.Time, time.Time, error) { } else if len(args) > 0 { return time.Time{}, time.Time{}, fmt.Errorf("cannot combine --from/--to with relative value: %s", args[0]) } - from, err := time.Parse(time.RFC3339, fromArg) + t1, err := time.Parse(time.RFC3339, from) if err != nil { return time.Time{}, time.Time{}, err } - to, err := time.Parse(time.RFC3339, toArg) + t2, err := time.Parse(time.RFC3339, to) if err != nil { return time.Time{}, time.Time{}, err } - if !to.After(from) { + if !t2.After(t1) { return time.Time{}, time.Time{}, fmt.Errorf("--to must specify a time after --from") } - return from, to, nil + return t1, t2, nil } diff --git a/client/go/cmd/log_test.go b/client/go/cmd/log_test.go index 3f6714b0d3c..4a6bd89c12d 100644 --- a/client/go/cmd/log_test.go +++ b/client/go/cmd/log_test.go @@ -2,52 +2,50 @@ package cmd import ( - "path/filepath" "testing" "github.com/stretchr/testify/assert" - "github.com/vespa-engine/vespa/client/go/build" "github.com/vespa-engine/vespa/client/go/mock" + "github.com/vespa-engine/vespa/client/go/version" ) func TestLog(t *testing.T) { - homeDir := filepath.Join(t.TempDir(), ".vespa") pkgDir := mockApplicationPackage(t, false) httpClient := &mock.HTTPClient{} httpClient.NextResponse(200, `1632738690.905535 host1a.dev.aws-us-east-1c 806/53 logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication info Switching to the latest deployed set of configurations and components. Application config generation: 52532`) - execute(command{homeDir: homeDir, args: []string{"config", "set", "application", "t1.a1.i1"}}, t, httpClient) - execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "cloud"}}, t, httpClient) - execute(command{homeDir: homeDir, args: []string{"auth", "api-key"}}, t, httpClient) - execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "cloud"}}, t, httpClient) - execute(command{homeDir: homeDir, args: []string{"config", "set", "api-key-file", filepath.Join(homeDir, "t1.api-key.pem")}}, t, httpClient) - execute(command{homeDir: homeDir, args: []string{"auth", "cert", pkgDir}}, t, httpClient) + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = httpClient - out, outErr := execute(command{homeDir: homeDir, args: []string{"log", "--from", "2021-09-27T10:00:00Z", "--to", "2021-09-27T11:00:00Z"}}, t, httpClient) - assert.Equal(t, "", outErr) + assert.Nil(t, cli.Run("config", "set", "application", "t1.a1.i1")) + assert.Nil(t, cli.Run("config", "set", "target", "cloud")) + assert.Nil(t, cli.Run("auth", "api-key")) + assert.Nil(t, cli.Run("auth", "cert", pkgDir)) + stdout.Reset() + assert.Nil(t, cli.Run("log", "--from", "2021-09-27T10:00:00Z", "--to", "2021-09-27T11:00:00Z")) expected := "[2021-09-27 10:31:30.905535] host1a.dev.aws-us-east-1c info logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication Switching to the latest deployed set of configurations and components. Application config generation: 52532\n" - assert.Equal(t, expected, out) + assert.Equal(t, expected, stdout.String()) - _, errOut := execute(command{homeDir: homeDir, args: []string{"log", "--from", "2021-09-27T13:12:49Z", "--to", "2021-09-27T13:15:00", "1h"}}, t, httpClient) - assert.Equal(t, "Error: invalid period: cannot combine --from/--to with relative value: 1h\n", errOut) + assert.NotNil(t, cli.Run("log", "--from", "2021-09-27T13:12:49Z", "--to", "2021-09-27T13:15:00", "1h")) + assert.Contains(t, stderr.String(), "Error: invalid period: cannot combine --from/--to with relative value: 1h\n") } func TestLogOldClient(t *testing.T) { - buildVersion := build.Version - build.Version = "7.0.0" - homeDir := filepath.Join(t.TempDir(), ".vespa") + cli, _, stderr := newTestCLI(t) + cli.version = version.MustParse("7.0.0") + pkgDir := mockApplicationPackage(t, false) httpClient := &mock.HTTPClient{} httpClient.NextResponse(200, `{"minVersion": "8.0.0"}`) - execute(command{homeDir: homeDir, args: []string{"config", "set", "application", "t1.a1.i1"}}, t, httpClient) - execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "cloud"}}, t, httpClient) - execute(command{homeDir: homeDir, args: []string{"auth", "api-key"}}, t, httpClient) - execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "cloud"}}, t, httpClient) - execute(command{homeDir: homeDir, args: []string{"config", "set", "api-key-file", filepath.Join(homeDir, "t1.api-key.pem")}}, t, httpClient) - execute(command{homeDir: homeDir, args: []string{"auth", "cert", pkgDir}}, t, httpClient) - out, errOut := execute(command{homeDir: homeDir, args: []string{"log"}}, t, httpClient) - assert.Equal(t, "", out) + httpClient.NextResponse(200, `1632738690.905535 host1a.dev.aws-us-east-1c 806/53 logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication info Switching to the latest deployed set of configurations and components. Application config generation: 52532`) + cli.httpClient = httpClient + + assert.Nil(t, cli.Run("config", "set", "application", "t1.a1.i1")) + assert.Nil(t, cli.Run("config", "set", "target", "cloud")) + assert.Nil(t, cli.Run("auth", "api-key")) + assert.Nil(t, cli.Run("auth", "cert", pkgDir)) + + assert.Nil(t, cli.Run("log")) expected := "Error: client version 7.0.0 is less than the minimum supported version: 8.0.0\nHint: This is not a fatal error, but this version may not work as expected\nHint: Try 'vespa version' to check for a new version\n" - assert.Equal(t, expected, errOut) - build.Version = buildVersion + assert.Contains(t, stderr.String(), expected) } diff --git a/client/go/cmd/login.go b/client/go/cmd/login.go index 2ac480d05f5..3750037be88 100644 --- a/client/go/cmd/login.go +++ b/client/go/cmd/login.go @@ -2,35 +2,33 @@ package cmd import ( "github.com/spf13/cobra" - "github.com/vespa-engine/vespa/client/go/auth0" + "github.com/vespa-engine/vespa/client/go/auth/auth0" ) -var loginCmd = &cobra.Command{ - Use: "login", - Args: cobra.NoArgs, - Short: "Authenticate the Vespa CLI", - Example: "$ vespa auth login", - DisableAutoGenTag: true, - SilenceUsage: true, - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - cfg, err := LoadConfig() - if err != nil { +func newLoginCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "login", + Args: cobra.NoArgs, + Short: "Authenticate the Vespa CLI", + Example: "$ vespa auth login", + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + targetType, err := cli.config.targetType() + if err != nil { + return err + } + system, err := cli.system(targetType) + if err != nil { + return err + } + a, err := auth0.GetAuth0(cli.config.authConfigPath(), system.Name, system.URL) + if err != nil { + return err + } + _, err = auth0.RunLogin(ctx, a, false) return err - } - targetType, err := getTargetType() - if err != nil { - return err - } - system, err := getSystem(targetType) - if err != nil { - return err - } - a, err := auth0.GetAuth0(cfg.AuthConfigPath(), system.Name, system.URL) - if err != nil { - return err - } - _, err = auth0.RunLogin(ctx, a, false) - return err - }, + }, + } } diff --git a/client/go/cmd/logout.go b/client/go/cmd/logout.go index b1f2477aba4..6cef5ee371c 100644 --- a/client/go/cmd/logout.go +++ b/client/go/cmd/logout.go @@ -2,34 +2,32 @@ package cmd import ( "github.com/spf13/cobra" - "github.com/vespa-engine/vespa/client/go/auth0" + "github.com/vespa-engine/vespa/client/go/auth/auth0" ) -var logoutCmd = &cobra.Command{ - Use: "logout", - Args: cobra.NoArgs, - Short: "Log out of Vespa Cli", - Example: "$ vespa auth logout", - DisableAutoGenTag: true, - SilenceUsage: true, - RunE: func(cmd *cobra.Command, args []string) error { - cfg, err := LoadConfig() - if err != nil { +func newLogoutCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "logout", + Args: cobra.NoArgs, + Short: "Log out of Vespa Cli", + Example: "$ vespa auth logout", + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + targetType, err := cli.config.targetType() + if err != nil { + return err + } + system, err := cli.system(targetType) + if err != nil { + return err + } + a, err := auth0.GetAuth0(cli.config.authConfigPath(), system.Name, system.URL) + if err != nil { + return err + } + err = auth0.RunLogout(a) return err - } - targetType, err := getTargetType() - if err != nil { - return err - } - system, err := getSystem(targetType) - if err != nil { - return err - } - a, err := auth0.GetAuth0(cfg.AuthConfigPath(), system.Name, system.URL) - if err != nil { - return err - } - err = auth0.RunLogout(a) - return err - }, + }, + } } diff --git a/client/go/cmd/man.go b/client/go/cmd/man.go index 01fffd38a32..4d139adb244 100644 --- a/client/go/cmd/man.go +++ b/client/go/cmd/man.go @@ -8,24 +8,22 @@ import ( "github.com/spf13/cobra/doc" ) -func init() { - rootCmd.AddCommand(manCmd) -} - -var manCmd = &cobra.Command{ - Use: "man directory", - Short: "Generate man pages and write them to given directory", - Args: cobra.ExactArgs(1), - Hidden: true, // Not intended to be called by users - DisableAutoGenTag: true, - SilenceUsage: true, - RunE: func(cmd *cobra.Command, args []string) error { - dir := args[0] - err := doc.GenManTree(rootCmd, nil, dir) - if err != nil { - return fmt.Errorf("failed to write man pages: %w", err) - } - printSuccess("Man pages written to ", dir) - return nil - }, +func newManCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "man directory", + Short: "Generate man pages and write them to given directory", + Args: cobra.ExactArgs(1), + Hidden: true, // Not intended to be called by users + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + dir := args[0] + err := doc.GenManTree(cli.cmd, nil, dir) + if err != nil { + return fmt.Errorf("failed to write man pages: %w", err) + } + cli.printSuccess("Man pages written to ", dir) + return nil + }, + } } diff --git a/client/go/cmd/man_test.go b/client/go/cmd/man_test.go index dfbe04f4c8e..b885d6a8fac 100644 --- a/client/go/cmd/man_test.go +++ b/client/go/cmd/man_test.go @@ -12,7 +12,8 @@ import ( func TestMan(t *testing.T) { tmpDir := t.TempDir() - out, _ := execute(command{args: []string{"man", tmpDir}}, t, nil) - assert.Equal(t, fmt.Sprintf("Success: Man pages written to %s\n", tmpDir), out) + cli, stdout, _ := newTestCLI(t) + assert.Nil(t, cli.Run("man", tmpDir)) + assert.Equal(t, fmt.Sprintf("Success: Man pages written to %s\n", tmpDir), stdout.String()) assert.True(t, util.PathExists(filepath.Join(tmpDir, "vespa.1"))) } diff --git a/client/go/cmd/prod.go b/client/go/cmd/prod.go index 4ce126bebb4..9108cb7a222 100644 --- a/client/go/cmd/prod.go +++ b/client/go/cmd/prod.go @@ -6,6 +6,7 @@ import ( "bytes" "errors" "fmt" + "io" "io/ioutil" "log" "os" @@ -19,32 +20,29 @@ import ( "github.com/vespa-engine/vespa/client/go/vespa/xml" ) -func init() { - rootCmd.AddCommand(prodCmd) - prodCmd.AddCommand(prodInitCmd) - prodCmd.AddCommand(prodSubmitCmd) -} - -var prodCmd = &cobra.Command{ - Use: "prod", - Short: "Deploy an application package to production in Vespa Cloud", - Long: `Deploy an application package to production in Vespa Cloud. +func newProdCmd() *cobra.Command { + return &cobra.Command{ + Use: "prod", + Short: "Deploy an application package to production in Vespa Cloud", + Long: `Deploy an application package to production in Vespa Cloud. Configure and deploy your application package to production in Vespa Cloud.`, - Example: `$ vespa prod init + Example: `$ vespa prod init $ vespa prod submit`, - DisableAutoGenTag: true, - SilenceUsage: false, - Args: cobra.MinimumNArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - return fmt.Errorf("invalid command: %s", args[0]) - }, + DisableAutoGenTag: true, + SilenceUsage: false, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return fmt.Errorf("invalid command: %s", args[0]) + }, + } } -var prodInitCmd = &cobra.Command{ - Use: "init", - Short: "Modify service.xml and deployment.xml for production deployment", - Long: `Modify service.xml and deployment.xml for production deployment. +func newProdInitCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "init", + Short: "Modify service.xml and deployment.xml for production deployment", + Long: `Modify service.xml and deployment.xml for production deployment. Only basic deployment configuration is available through this command. For advanced configuration see the relevant Vespa Cloud documentation and make @@ -53,62 +51,64 @@ changes to deployment.xml and services.xml directly. Reference: https://cloud.vespa.ai/en/reference/services https://cloud.vespa.ai/en/reference/deployment`, - DisableAutoGenTag: true, - SilenceUsage: true, - RunE: func(cmd *cobra.Command, args []string) error { - appSource := applicationSource(args) - pkg, err := vespa.FindApplicationPackage(appSource, false) - if err != nil { - return err - } - if pkg.IsZip() { - return errHint(fmt.Errorf("cannot modify compressed application package %s", pkg.Path), - "Try running 'mvn clean' and run this command again") - } + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + appSource := applicationSource(args) + pkg, err := vespa.FindApplicationPackage(appSource, false) + if err != nil { + return err + } + if pkg.IsZip() { + return errHint(fmt.Errorf("cannot modify compressed application package %s", pkg.Path), + "Try running 'mvn clean' and run this command again") + } - deploymentXML, err := readDeploymentXML(pkg) - if err != nil { - return fmt.Errorf("could not read deployment.xml: %w", err) - } - servicesXML, err := readServicesXML(pkg) - if err != nil { - return fmt.Errorf("a services.xml declaring your cluster(s) must exist: %w", err) - } - target, err := getTarget() - if err != nil { - return err - } + deploymentXML, err := readDeploymentXML(pkg) + if err != nil { + return fmt.Errorf("could not read deployment.xml: %w", err) + } + servicesXML, err := readServicesXML(pkg) + if err != nil { + return fmt.Errorf("a services.xml declaring your cluster(s) must exist: %w", err) + } + target, err := cli.target(targetOptions{noCertificate: true}) + if err != nil { + return err + } - fmt.Fprint(stdout, "This will modify any existing ", color.YellowString("deployment.xml"), " and ", color.YellowString("services.xml"), - "!\nBefore modification a backup of the original file will be created.\n\n") - fmt.Fprint(stdout, "A default value is suggested (shown inside brackets) based on\nthe files' existing contents. Press enter to use it.\n\n") - fmt.Fprint(stdout, "Abort the configuration at any time by pressing Ctrl-C. The\nfiles will remain untouched.\n\n") - fmt.Fprint(stdout, "See this guide for sizing a Vespa deployment:\n", color.GreenString("https://docs.vespa.ai/en/performance/sizing-search.html\n\n")) - r := bufio.NewReader(stdin) - deploymentXML, err = updateRegions(r, deploymentXML, target.Deployment().System) - if err != nil { - return err - } - servicesXML, err = updateNodes(r, servicesXML) - if err != nil { - return err - } + fmt.Fprint(cli.Stdout, "This will modify any existing ", color.YellowString("deployment.xml"), " and ", color.YellowString("services.xml"), + "!\nBefore modification a backup of the original file will be created.\n\n") + fmt.Fprint(cli.Stdout, "A default value is suggested (shown inside brackets) based on\nthe files' existing contents. Press enter to use it.\n\n") + fmt.Fprint(cli.Stdout, "Abort the configuration at any time by pressing Ctrl-C. The\nfiles will remain untouched.\n\n") + fmt.Fprint(cli.Stdout, "See this guide for sizing a Vespa deployment:\n", color.GreenString("https://docs.vespa.ai/en/performance/sizing-search.html\n\n")) + r := bufio.NewReader(cli.Stdin) + deploymentXML, err = updateRegions(cli, r, deploymentXML, target.Deployment().System) + if err != nil { + return err + } + servicesXML, err = updateNodes(cli, r, servicesXML) + if err != nil { + return err + } - fmt.Fprintln(stdout) - if err := writeWithBackup(pkg, "deployment.xml", deploymentXML.String()); err != nil { - return err - } - if err := writeWithBackup(pkg, "services.xml", servicesXML.String()); err != nil { - return err - } - return nil - }, + fmt.Fprintln(cli.Stdout) + if err := writeWithBackup(cli.Stdout, pkg, "deployment.xml", deploymentXML.String()); err != nil { + return err + } + if err := writeWithBackup(cli.Stdout, pkg, "services.xml", servicesXML.String()); err != nil { + return err + } + return nil + }, + } } -var prodSubmitCmd = &cobra.Command{ - Use: "submit", - Short: "Submit your application for production deployment", - Long: `Submit your application for production deployment. +func newProdSubmitCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "submit", + Short: "Submit your application for production deployment", + Long: `Submit your application for production deployment. This commands uploads your application package to Vespa Cloud and deploys it to the production zones specified in deployment.xml. @@ -123,58 +123,52 @@ by a continuous build system. For more information about production deployments in Vespa Cloud see: https://cloud.vespa.ai/en/getting-to-production https://cloud.vespa.ai/en/automated-deployments`, - DisableAutoGenTag: true, - SilenceUsage: true, - Example: `$ mvn package # when adding custom Java components + DisableAutoGenTag: true, + SilenceUsage: true, + Example: `$ mvn package # when adding custom Java components $ vespa prod submit`, - RunE: func(cmd *cobra.Command, args []string) error { - target, err := getTarget() - if err != nil { - return err - } - if target.Type() != vespa.TargetCloud { - // TODO: Add support for hosted - return fmt.Errorf("prod submit does not support %s target", target.Type()) - } - appSource := applicationSource(args) - pkg, err := vespa.FindApplicationPackage(appSource, true) - if err != nil { - return err - } - cfg, err := LoadConfig() - if err != nil { - return err - } - if !pkg.HasDeployment() { - return errHint(fmt.Errorf("no deployment.xml found"), "Try creating one with vespa prod init") - } - if pkg.TestPath == "" { - return errHint(fmt.Errorf("no tests found"), - "The application must be a Java maven project, or include basic HTTP tests under src/test/application/", - "See https://cloud.vespa.ai/en/getting-to-production") - } - if err := verifyTests(pkg, target); err != nil { - return err - } - if !isCI() { - printWarning("We recommend doing this only from a CD job", "See https://cloud.vespa.ai/en/getting-to-production") - } - opts, err := getDeploymentOptions(cfg, pkg, target) - if err != nil { - return err - } - if err := vespa.Submit(opts); err != nil { - return fmt.Errorf("could not submit application for deployment: %w", err) - } else { - printSuccess("Submitted ", color.CyanString(pkg.Path), " for deployment") - log.Printf("See %s for deployment progress\n", color.CyanString(fmt.Sprintf("%s/tenant/%s/application/%s/prod/deployment", - opts.Target.Deployment().System.ConsoleURL, opts.Target.Deployment().Application.Tenant, opts.Target.Deployment().Application.Application))) - } - return nil - }, + RunE: func(cmd *cobra.Command, args []string) error { + target, err := cli.target(targetOptions{noCertificate: true}) + if err != nil { + return err + } + if target.Type() != vespa.TargetCloud { + // TODO: Add support for hosted + return fmt.Errorf("prod submit does not support %s target", target.Type()) + } + appSource := applicationSource(args) + pkg, err := vespa.FindApplicationPackage(appSource, true) + if err != nil { + return err + } + if !pkg.HasDeployment() { + return errHint(fmt.Errorf("no deployment.xml found"), "Try creating one with vespa prod init") + } + if pkg.TestPath == "" { + return errHint(fmt.Errorf("no tests found"), + "The application must be a Java maven project, or include basic HTTP tests under src/test/application/", + "See https://cloud.vespa.ai/en/getting-to-production") + } + if err := verifyTests(cli, pkg); err != nil { + return err + } + if !cli.isCI() { + cli.printWarning("We recommend doing this only from a CD job", "See https://cloud.vespa.ai/en/getting-to-production") + } + opts := cli.createDeploymentOptions(pkg, target) + if err := vespa.Submit(opts); err != nil { + return fmt.Errorf("could not submit application for deployment: %w", err) + } else { + cli.printSuccess("Submitted ", color.CyanString(pkg.Path), " for deployment") + log.Printf("See %s for deployment progress\n", color.CyanString(fmt.Sprintf("%s/tenant/%s/application/%s/prod/deployment", + opts.Target.Deployment().System.ConsoleURL, opts.Target.Deployment().Application.Tenant, opts.Target.Deployment().Application.Application))) + } + return nil + }, + } } -func writeWithBackup(pkg vespa.ApplicationPackage, filename, contents string) error { +func writeWithBackup(stdout io.Writer, pkg vespa.ApplicationPackage, filename, contents string) error { dst := filepath.Join(pkg.Path, filename) if util.PathExists(dst) { data, err := ioutil.ReadFile(dst) @@ -205,8 +199,8 @@ func writeWithBackup(pkg vespa.ApplicationPackage, filename, contents string) er return ioutil.WriteFile(dst, []byte(contents), 0644) } -func updateRegions(r *bufio.Reader, deploymentXML xml.Deployment, system vespa.System) (xml.Deployment, error) { - regions, err := promptRegions(r, deploymentXML, system) +func updateRegions(cli *CLI, stdin *bufio.Reader, deploymentXML xml.Deployment, system vespa.System) (xml.Deployment, error) { + regions, err := promptRegions(cli, stdin, deploymentXML, system) if err != nil { return xml.Deployment{}, err } @@ -225,10 +219,10 @@ func updateRegions(r *bufio.Reader, deploymentXML xml.Deployment, system vespa.S return deploymentXML, nil } -func promptRegions(r *bufio.Reader, deploymentXML xml.Deployment, system vespa.System) (string, error) { - fmt.Fprintln(stdout, color.CyanString("> Deployment regions")) - fmt.Fprintf(stdout, "Documentation: %s\n", color.GreenString("https://cloud.vespa.ai/en/reference/zones")) - fmt.Fprintf(stdout, "Example: %s\n\n", color.YellowString("aws-us-east-1c,aws-us-west-2a")) +func promptRegions(cli *CLI, stdin *bufio.Reader, deploymentXML xml.Deployment, system vespa.System) (string, error) { + fmt.Fprintln(cli.Stdout, color.CyanString("> Deployment regions")) + fmt.Fprintf(cli.Stdout, "Documentation: %s\n", color.GreenString("https://cloud.vespa.ai/en/reference/zones")) + fmt.Fprintf(cli.Stdout, "Example: %s\n\n", color.YellowString("aws-us-east-1c,aws-us-west-2a")) var currentRegions []string for _, r := range deploymentXML.Prod.Regions { currentRegions = append(currentRegions, r.Name) @@ -247,12 +241,12 @@ func promptRegions(r *bufio.Reader, deploymentXML xml.Deployment, system vespa.S } return nil } - return prompt(r, "Which regions do you wish to deploy in?", strings.Join(currentRegions, ","), validator) + return prompt(cli, stdin, "Which regions do you wish to deploy in?", strings.Join(currentRegions, ","), validator) } -func updateNodes(r *bufio.Reader, servicesXML xml.Services) (xml.Services, error) { +func updateNodes(cli *CLI, r *bufio.Reader, servicesXML xml.Services) (xml.Services, error) { for _, c := range servicesXML.Container { - nodes, err := promptNodes(r, c.ID, c.Nodes) + nodes, err := promptNodes(cli, r, c.ID, c.Nodes) if err != nil { return xml.Services{}, err } @@ -261,7 +255,7 @@ func updateNodes(r *bufio.Reader, servicesXML xml.Services) (xml.Services, error } } for _, c := range servicesXML.Content { - nodes, err := promptNodes(r, c.ID, c.Nodes) + nodes, err := promptNodes(cli, r, c.ID, c.Nodes) if err != nil { return xml.Services{}, err } @@ -272,8 +266,8 @@ func updateNodes(r *bufio.Reader, servicesXML xml.Services) (xml.Services, error return servicesXML, nil } -func promptNodes(r *bufio.Reader, clusterID string, defaultValue xml.Nodes) (xml.Nodes, error) { - count, err := promptNodeCount(r, clusterID, defaultValue.Count) +func promptNodes(cli *CLI, r *bufio.Reader, clusterID string, defaultValue xml.Nodes) (xml.Nodes, error) { + count, err := promptNodeCount(cli, r, clusterID, defaultValue.Count) if err != nil { return xml.Nodes{}, err } @@ -283,7 +277,7 @@ func promptNodes(r *bufio.Reader, clusterID string, defaultValue xml.Nodes) (xml if resources != nil { defaultSpec = defaultValue.Resources.String() } - spec, err := promptResources(r, clusterID, defaultSpec) + spec, err := promptResources(cli, r, clusterID, defaultSpec) if err != nil { return xml.Nodes{}, err } @@ -299,21 +293,21 @@ func promptNodes(r *bufio.Reader, clusterID string, defaultValue xml.Nodes) (xml return xml.Nodes{Count: count, Resources: resources}, nil } -func promptNodeCount(r *bufio.Reader, clusterID string, nodeCount string) (string, error) { - fmt.Fprintln(stdout, color.CyanString("\n> Node count: "+clusterID+" cluster")) - fmt.Fprintf(stdout, "Documentation: %s\n", color.GreenString("https://cloud.vespa.ai/en/reference/services")) - fmt.Fprintf(stdout, "Example: %s\nExample: %s\n\n", color.YellowString("4"), color.YellowString("[2,8]")) +func promptNodeCount(cli *CLI, stdin *bufio.Reader, clusterID string, nodeCount string) (string, error) { + fmt.Fprintln(cli.Stdout, color.CyanString("\n> Node count: "+clusterID+" cluster")) + fmt.Fprintf(cli.Stdout, "Documentation: %s\n", color.GreenString("https://cloud.vespa.ai/en/reference/services")) + fmt.Fprintf(cli.Stdout, "Example: %s\nExample: %s\n\n", color.YellowString("4"), color.YellowString("[2,8]")) validator := func(input string) error { _, _, err := xml.ParseNodeCount(input) return err } - return prompt(r, fmt.Sprintf("How many nodes should the %s cluster have?", color.CyanString(clusterID)), nodeCount, validator) + return prompt(cli, stdin, fmt.Sprintf("How many nodes should the %s cluster have?", color.CyanString(clusterID)), nodeCount, validator) } -func promptResources(r *bufio.Reader, clusterID string, resources string) (string, error) { - fmt.Fprintln(stdout, color.CyanString("\n> Node resources: "+clusterID+" cluster")) - fmt.Fprintf(stdout, "Documentation: %s\n", color.GreenString("https://cloud.vespa.ai/en/reference/services")) - fmt.Fprintf(stdout, "Example: %s\nExample: %s\n\n", color.YellowString("auto"), color.YellowString("vcpu=4,memory=8Gb,disk=100Gb")) +func promptResources(cli *CLI, stdin *bufio.Reader, clusterID string, resources string) (string, error) { + fmt.Fprintln(cli.Stdout, color.CyanString("\n> Node resources: "+clusterID+" cluster")) + fmt.Fprintf(cli.Stdout, "Documentation: %s\n", color.GreenString("https://cloud.vespa.ai/en/reference/services")) + fmt.Fprintf(cli.Stdout, "Example: %s\nExample: %s\n\n", color.YellowString("auto"), color.YellowString("vcpu=4,memory=8Gb,disk=100Gb")) validator := func(input string) error { if input == "auto" { return nil @@ -321,7 +315,7 @@ func promptResources(r *bufio.Reader, clusterID string, resources string) (strin _, err := xml.ParseResources(input) return err } - return prompt(r, fmt.Sprintf("Which resources should each node in the %s cluster have?", color.CyanString(clusterID)), resources, validator) + return prompt(cli, stdin, fmt.Sprintf("Which resources should each node in the %s cluster have?", color.CyanString(clusterID)), resources, validator) } func readDeploymentXML(pkg vespa.ApplicationPackage) (xml.Deployment, error) { @@ -345,17 +339,17 @@ func readServicesXML(pkg vespa.ApplicationPackage) (xml.Services, error) { return xml.ReadServices(f) } -func prompt(r *bufio.Reader, question, defaultAnswer string, validator func(input string) error) (string, error) { +func prompt(cli *CLI, stdin *bufio.Reader, question, defaultAnswer string, validator func(input string) error) (string, error) { var input string for input == "" { - fmt.Fprint(stdout, question) + fmt.Fprint(cli.Stdout, question) if defaultAnswer != "" { - fmt.Fprint(stdout, " [", color.YellowString(defaultAnswer), "]") + fmt.Fprint(cli.Stdout, " [", color.YellowString(defaultAnswer), "]") } - fmt.Fprint(stdout, " ") + fmt.Fprint(cli.Stdout, " ") var err error - input, err = r.ReadString('\n') + input, err = stdin.ReadString('\n') if err != nil { return "", err } @@ -365,15 +359,15 @@ func prompt(r *bufio.Reader, question, defaultAnswer string, validator func(inpu } if err := validator(input); err != nil { - printErrHint(err) - fmt.Fprintln(stderr) + cli.printErrHint(err) + fmt.Fprintln(cli.Stderr) input = "" } } return input, nil } -func verifyTests(app vespa.ApplicationPackage, target vespa.Target) error { +func verifyTests(cli *CLI, app vespa.ApplicationPackage) error { // TODO: system-test, staging-setup and staging-test should be required if the application // does not have any Java tests. suites := map[string]bool{ @@ -392,14 +386,14 @@ func verifyTests(app vespa.ApplicationPackage, target vespa.Target) error { testPath = path } for suite, required := range suites { - if err := verifyTest(testPath, suite, target, required); err != nil { + if err := verifyTest(cli, testPath, suite, required); err != nil { return err } } return nil } -func verifyTest(testsParent string, suite string, target vespa.Target, required bool) error { +func verifyTest(cli *CLI, testsParent string, suite string, required bool) error { testDirectory := filepath.Join(testsParent, "tests", suite) _, err := os.Stat(testDirectory) if err != nil { @@ -413,6 +407,6 @@ func verifyTest(testsParent string, suite string, target vespa.Target, required } return nil } - _, _, err = runTests(testDirectory, true) + _, _, err = runTests(cli, "", testDirectory, true) return err } diff --git a/client/go/cmd/prod_test.go b/client/go/cmd/prod_test.go index 99ec57c945f..06883dfcdcf 100644 --- a/client/go/cmd/prod_test.go +++ b/client/go/cmd/prod_test.go @@ -10,12 +10,13 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/vespa-engine/vespa/client/go/mock" "github.com/vespa-engine/vespa/client/go/util" + "github.com/vespa-engine/vespa/client/go/vespa" ) func TestProdInit(t *testing.T) { - homeDir := filepath.Join(t.TempDir(), ".vespa") pkgDir := filepath.Join(t.TempDir(), "app") createApplication(t, pkgDir, false) @@ -42,7 +43,10 @@ func TestProdInit(t *testing.T) { } var buf bytes.Buffer buf.WriteString(strings.Join(answers, "\n") + "\n") - execute(command{stdin: &buf, homeDir: homeDir, args: []string{"prod", "init", pkgDir}}, t, nil) + + cli, _, _ := newTestCLI(t) + cli.Stdin = &buf + assert.Nil(t, cli.Run("prod", "init", pkgDir)) // Verify contents deploymentPath := filepath.Join(pkgDir, "src", "main", "application", "deployment.xml") @@ -144,18 +148,33 @@ func writeTest(path string, content []byte, t *testing.T) { } func TestProdSubmit(t *testing.T) { - homeDir := filepath.Join(t.TempDir(), ".vespa") pkgDir := filepath.Join(t.TempDir(), "app") createApplication(t, pkgDir, false) httpClient := &mock.HTTPClient{} httpClient.NextResponse(200, `ok`) - execute(command{homeDir: homeDir, args: []string{"config", "set", "application", "t1.a1.i1"}}, t, httpClient) - execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "cloud"}}, t, httpClient) - execute(command{homeDir: homeDir, args: []string{"auth", "api-key"}}, t, httpClient) - execute(command{homeDir: homeDir, args: []string{"auth", "cert", pkgDir}}, t, httpClient) - // Zipping requires relative paths, so much let command run from pkgDir, then reset cwd for subsequent tests. + cli, stdout, _ := newTestCLI(t, "CI=true") + cli.httpClient = httpClient + app := vespa.ApplicationID{Tenant: "t1", Application: "a1", Instance: "i1"} + assert.Nil(t, cli.Run("config", "set", "application", app.String())) + assert.Nil(t, cli.Run("config", "set", "target", "cloud")) + assert.Nil(t, cli.Run("auth", "api-key")) + assert.Nil(t, cli.Run("auth", "cert", pkgDir)) + + // Remove certificate as it's not required for submission (but it must be part of the application package) + if path, err := cli.config.privateKeyPath(app); err == nil { + os.RemoveAll(path) + } else { + require.Nil(t, err) + } + if path, err := cli.config.certificatePath(app); err == nil { + os.RemoveAll(path) + } else { + require.Nil(t, err) + } + + // Zipping requires relative paths, so must let command run from pkgDir, then reset cwd for subsequent tests. if cwd, err := os.Getwd(); err != nil { t.Fatal(err) } else { @@ -164,26 +183,25 @@ func TestProdSubmit(t *testing.T) { if err := os.Chdir(pkgDir); err != nil { t.Fatal(err) } - if err := os.Setenv("CI", "true"); err != nil { - t.Fatal(err) - } - out, outErr := execute(command{homeDir: homeDir, args: []string{"prod", "submit", "-k", filepath.Join(homeDir, "t1.api-key.pem")}}, t, httpClient) - assert.Equal(t, "", outErr) - assert.Contains(t, out, "Success: Submitted") - assert.Contains(t, out, "See https://console.vespa-cloud.com/tenant/t1/application/a1/prod/deployment for deployment progress") + + stdout.Reset() + assert.Nil(t, cli.Run("prod", "submit", "-k", filepath.Join(cli.config.homeDir, "t1.api-key.pem"))) + assert.Contains(t, stdout.String(), "Success: Submitted") + assert.Contains(t, stdout.String(), "See https://console.vespa-cloud.com/tenant/t1/application/a1/prod/deployment for deployment progress") } func TestProdSubmitWithJava(t *testing.T) { - homeDir := filepath.Join(t.TempDir(), ".vespa") pkgDir := filepath.Join(t.TempDir(), "app") createApplication(t, pkgDir, true) httpClient := &mock.HTTPClient{} httpClient.NextResponse(200, `ok`) - execute(command{homeDir: homeDir, args: []string{"config", "set", "application", "t1.a1.i1"}}, t, httpClient) - execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "cloud"}}, t, httpClient) - execute(command{homeDir: homeDir, args: []string{"auth", "api-key"}}, t, httpClient) - execute(command{homeDir: homeDir, args: []string{"auth", "cert", pkgDir}}, t, httpClient) + cli, stdout, _ := newTestCLI(t, "CI=true") + cli.httpClient = httpClient + assert.Nil(t, cli.Run("config", "set", "application", "t1.a1.i1")) + assert.Nil(t, cli.Run("config", "set", "target", "cloud")) + assert.Nil(t, cli.Run("auth", "api-key")) + assert.Nil(t, cli.Run("auth", "cert", pkgDir)) // Copy an application package pre-assembled with mvn package testAppDir := filepath.Join("testdata", "applications", "withDeployment", "target") @@ -192,10 +210,10 @@ func TestProdSubmitWithJava(t *testing.T) { testZipFile := filepath.Join(testAppDir, "application-test.zip") copyFile(t, filepath.Join(pkgDir, "target", "application-test.zip"), testZipFile) - out, outErr := execute(command{homeDir: homeDir, args: []string{"prod", "submit", "-k", filepath.Join(homeDir, "t1.api-key.pem"), pkgDir}}, t, httpClient) - assert.Equal(t, "", outErr) - assert.Contains(t, out, "Success: Submitted") - assert.Contains(t, out, "See https://console.vespa-cloud.com/tenant/t1/application/a1/prod/deployment for deployment progress") + stdout.Reset() + assert.Nil(t, cli.Run("prod", "submit", "-k", filepath.Join(cli.config.homeDir, "t1.api-key.pem"), pkgDir)) + assert.Contains(t, stdout.String(), "Success: Submitted") + assert.Contains(t, stdout.String(), "See https://console.vespa-cloud.com/tenant/t1/application/a1/prod/deployment for deployment progress") } func copyFile(t *testing.T, dstFilename, srcFilename string) { diff --git a/client/go/cmd/query.go b/client/go/cmd/query.go index cb0fd923c4e..c5868e2f71c 100644 --- a/client/go/cmd/query.go +++ b/client/go/cmd/query.go @@ -7,7 +7,6 @@ package cmd import ( "fmt" "io" - "io/ioutil" "log" "net/http" "net/url" @@ -21,49 +20,45 @@ import ( "github.com/vespa-engine/vespa/client/go/vespa" ) -var ( - queryPrintCurl bool - queryTimeoutSecs int -) - -func init() { - rootCmd.AddCommand(queryCmd) - queryCmd.PersistentFlags().BoolVarP(&queryPrintCurl, "verbose", "v", false, "Print the equivalent curl command for the query") - queryCmd.Flags().IntVarP(&queryTimeoutSecs, "timeout", "T", 10, "Timeout for the query in seconds") -} - -var queryCmd = &cobra.Command{ - Use: "query query-parameters", - Short: "Issue a query to Vespa", - Example: `$ vespa query "yql=select * from music where album contains 'head';" hits=5`, - Long: `Issue a query to Vespa. +func newQueryCmd(cli *CLI) *cobra.Command { + var ( + printCurl bool + queryTimeoutSecs int + ) + cmd := &cobra.Command{ + Use: "query query-parameters", + Short: "Issue a query to Vespa", + Example: `$ vespa query "yql=select * from music where album contains 'head';" hits=5`, + Long: `Issue a query to Vespa. Any parameter from https://docs.vespa.ai/en/reference/query-api-reference.html can be set by the syntax [parameter-name]=[value].`, - // TODO: Support referencing a query json file - DisableAutoGenTag: true, - SilenceUsage: true, - Args: cobra.MinimumNArgs(1), - RunE: query, + // TODO: Support referencing a query json file + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return query(cli, args, queryTimeoutSecs, printCurl) + }, + } + cmd.PersistentFlags().BoolVarP(&printCurl, "verbose", "v", false, "Print the equivalent curl command for the query") + cmd.Flags().IntVarP(&queryTimeoutSecs, "timeout", "T", 10, "Timeout for the query in seconds") + return cmd } -func printCurl(url string, service *vespa.Service) error { - out := ioutil.Discard - if queryPrintCurl { - out = stderr - } +func printCurl(stderr io.Writer, url string, service *vespa.Service) error { cmd, err := curl.RawArgs(url) if err != nil { return err } cmd.Certificate = service.TLSOptions.CertificateFile cmd.PrivateKey = service.TLSOptions.PrivateKeyFile - _, err = io.WriteString(out, cmd.String()+"\n") + _, err = io.WriteString(stderr, cmd.String()+"\n") return err } -func query(cmd *cobra.Command, arguments []string) error { - service, err := getService(vespa.QueryService, 0, "") +func query(cli *CLI, arguments []string, timeoutSecs int, curl bool) error { + service, err := cli.service(vespa.QueryService, 0, "") if err != nil { return err } @@ -76,7 +71,7 @@ func query(cmd *cobra.Command, arguments []string) error { queryTimeout := urlQuery.Get("timeout") if queryTimeout == "" { // No timeout set by user, use the timeout option - queryTimeout = fmt.Sprintf("%ds", queryTimeoutSecs) + queryTimeout = fmt.Sprintf("%ds", timeoutSecs) urlQuery.Set("timeout", queryTimeout) } url.RawQuery = urlQuery.Encode() @@ -84,8 +79,10 @@ func query(cmd *cobra.Command, arguments []string) error { if err != nil { return fmt.Errorf("invalid query timeout: %w", err) } - if err := printCurl(url.String(), service); err != nil { - return err + if curl { + if err := printCurl(cli.Stderr, url.String(), service); err != nil { + return err + } } response, err := service.Do(&http.Request{URL: url}, deadline+time.Second) // Slightly longer than query timeout if err != nil { diff --git a/client/go/cmd/query_test.go b/client/go/cmd/query_test.go index d57268b248e..a9348f8ddeb 100644 --- a/client/go/cmd/query_test.go +++ b/client/go/cmd/query_test.go @@ -22,10 +22,13 @@ func TestQuery(t *testing.T) { func TestQueryVerbose(t *testing.T) { client := &mock.HTTPClient{} client.NextResponse(200, "{\"query\":\"result\"}") - cmd := command{args: []string{"query", "-v", "select from sources * where title contains 'foo'"}} - out, errOut := execute(cmd, t, client) - assert.Equal(t, "curl http://127.0.0.1:8080/search/\\?timeout=10s\\&yql=select+from+sources+%2A+where+title+contains+%27foo%27\n", errOut) - assert.Equal(t, "{\n \"query\": \"result\"\n}\n", out) + + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = client + + assert.Nil(t, cli.Run("query", "-v", "select from sources * where title contains 'foo'")) + assert.Equal(t, "curl http://127.0.0.1:8080/search/\\?timeout=10s\\&yql=select+from+sources+%2A+where+title+contains+%27foo%27\n", stderr.String()) + assert.Equal(t, "{\n \"query\": \"result\"\n}\n", stdout.String()) } func TestQueryNonJsonResult(t *testing.T) { @@ -57,9 +60,14 @@ func TestServerError(t *testing.T) { func assertQuery(t *testing.T, expectedQuery string, query ...string) { client := &mock.HTTPClient{} client.NextResponse(200, "{\"query\":\"result\"}") + cli, stdout, _ := newTestCLI(t) + cli.httpClient = client + + args := []string{"query"} + assert.Nil(t, cli.Run(append(args, query...)...)) assert.Equal(t, "{\n \"query\": \"result\"\n}\n", - executeCommand(t, client, []string{"query"}, query), + stdout.String(), "query output") queryURL, err := queryServiceURL(client) require.Nil(t, err) @@ -69,27 +77,27 @@ func assertQuery(t *testing.T, expectedQuery string, query ...string) { func assertQueryError(t *testing.T, status int, errorMessage string) { client := &mock.HTTPClient{} client.NextResponse(status, errorMessage) - _, outErr := execute(command{args: []string{"query", "yql=select from sources * where title contains 'foo'"}}, t, client) + cli, _, stderr := newTestCLI(t) + cli.httpClient = client + assert.NotNil(t, cli.Run("query", "yql=select from sources * where title contains 'foo'")) assert.Equal(t, "Error: invalid query: Status "+strconv.Itoa(status)+"\n"+errorMessage+"\n", - outErr, + stderr.String(), "error output") } func assertQueryServiceError(t *testing.T, status int, errorMessage string) { client := &mock.HTTPClient{} client.NextResponse(status, errorMessage) - _, outErr := execute(command{args: []string{"query", "yql=select from sources * where title contains 'foo'"}}, t, client) + cli, _, stderr := newTestCLI(t) + cli.httpClient = client + assert.NotNil(t, cli.Run("query", "yql=select from sources * where title contains 'foo'")) assert.Equal(t, "Error: Status "+strconv.Itoa(status)+" from container at 127.0.0.1:8080\n"+errorMessage+"\n", - outErr, + stderr.String(), "error output") } func queryServiceURL(client *mock.HTTPClient) (string, error) { - service, err := getService("query", 0, "") - if err != nil { - return "", err - } - return service.BaseURL, nil + return "http://127.0.0.1:8080", nil } diff --git a/client/go/cmd/root.go b/client/go/cmd/root.go index 6f29a09e177..2dbc0e89e0f 100644 --- a/client/go/cmd/root.go +++ b/client/go/cmd/root.go @@ -1,22 +1,69 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Root Cobra command: vespa -// author: bratseth - package cmd import ( + "crypto/tls" + "crypto/x509" + "encoding/json" "fmt" "io" "io/ioutil" "log" "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" "github.com/fatih/color" "github.com/mattn/go-colorable" "github.com/mattn/go-isatty" "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/build" + "github.com/vespa-engine/vespa/client/go/util" + "github.com/vespa-engine/vespa/client/go/version" + "github.com/vespa-engine/vespa/client/go/vespa" ) +const ( + applicationFlag = "application" + targetFlag = "target" + waitFlag = "wait" + colorFlag = "color" + quietFlag = "quiet" + apiKeyFileFlag = "api-key-file" + apiKeyFlag = "api-key" +) + +// CLI holds the Vespa CLI command tree, configuration and dependencies. +type CLI struct { + // Environment holds the process environment. + Environment map[string]string + Stdin io.ReadWriter + Stdout io.Writer + Stderr io.Writer + + cmd *cobra.Command + flags *Flags + config *Config + version version.Version + + httpClient util.HTTPClient + exec executor + isTerminal func() bool +} + +// Flags holds the global Flags of Vespa CLI. +type Flags struct { + target string + application string + waitSecs int + color string + quiet bool + apiKeyFile string +} + // ErrCLI is an error returned to the user. It wraps an exit status, a regular error and optional hints for resolving // the error. type ErrCLI struct { @@ -26,8 +73,33 @@ type ErrCLI struct { error } -var ( - rootCmd = &cobra.Command{ +type targetOptions struct { + // zone declares the zone use when using this target. If empty, a default zone for the system is chosen. + zone string + // logLevel sets the log level to use for this target. If empty, it defaults to "info". + logLevel string + // noCertificate declares that no client certificate should be required when using this target. + noCertificate bool +} + +// errHint creates a new CLI error, with optional hints that will be printed after the error +func errHint(err error, hints ...string) ErrCLI { return ErrCLI{Status: 1, hints: hints, error: err} } + +type executor interface { + LookPath(name string) (string, error) + Run(name string, args ...string) ([]byte, error) +} + +type execSubprocess struct{} + +func (c *execSubprocess) LookPath(name string) (string, error) { return exec.LookPath(name) } +func (c *execSubprocess) Run(name string, args ...string) ([]byte, error) { + return exec.Command(name, args...).Output() +} + +// New creates the Vespa CLI, writing output to stdout and stderr, and reading environment variables from environment. +func New(stdout, stderr io.Writer, environment []string) (*CLI, error) { + cmd := &cobra.Command{ Use: "vespa command-name", Short: "The command-line tool for Vespa.ai", Long: `The command-line tool for Vespa.ai. @@ -39,63 +111,77 @@ Vespa documentation: https://docs.vespa.ai`, DisableAutoGenTag: true, SilenceErrors: true, // We have our own error printing SilenceUsage: false, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - return configureOutput() - }, - Args: cobra.MinimumNArgs(1), + Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { return fmt.Errorf("invalid command: %s", args[0]) }, } + env := make(map[string]string) + for _, entry := range environment { + parts := strings.SplitN(entry, "=", 2) + env[parts[0]] = parts[1] + } + version, err := version.Parse(build.Version) + if err != nil { + return nil, err + } + cli := CLI{ + Environment: env, + Stdin: os.Stdin, + Stdout: stdout, + Stderr: stderr, - targetArg string - applicationArg string - waitSecsArg int - colorArg string - quietArg bool - apiKeyFileArg string - stdin io.ReadWriter = os.Stdin - - stdout = colorable.NewColorableStdout() - stderr = colorable.NewColorableStderr() -) - -const ( - applicationFlag = "application" - targetFlag = "target" - waitFlag = "wait" - colorFlag = "color" - quietFlag = "quiet" - apiKeyFileFlag = "api-key-file" - apiKeyFlag = "api-key" -) - -func isTerminal() bool { - if f, ok := stdout.(*os.File); ok { - return isatty.IsTerminal(f.Fd()) + version: version, + cmd: cmd, + httpClient: util.CreateClient(time.Second * 10), + exec: &execSubprocess{}, } - if f, ok := stderr.(*os.File); ok { - return isatty.IsTerminal(f.Fd()) + cli.isTerminal = func() bool { return isTerminal(cli.Stdout) || isTerminal(cli.Stderr) } + cli.configureFlags() + if err := cli.loadConfig(); err != nil { + return nil, err } - return false + cli.configureCommands() + cmd.PersistentPreRunE = cli.configureOutput + return &cli, nil } -func configureOutput() error { - config, err := LoadConfig() +func (c *CLI) loadConfig() error { + bindings := NewConfigBindings() + bindings.bindFlag(targetFlag, c.cmd) + bindings.bindFlag(applicationFlag, c.cmd) + bindings.bindFlag(waitFlag, c.cmd) + bindings.bindFlag(colorFlag, c.cmd) + bindings.bindFlag(quietFlag, c.cmd) + bindings.bindFlag(apiKeyFileFlag, c.cmd) + bindings.bindEnvironment(apiKeyFlag, "VESPA_CLI_API_KEY") + bindings.bindEnvironment(apiKeyFileFlag, "VESPA_CLI_API_KEY_FILE") + config, err := loadConfig(c.Environment, bindings) if err != nil { return err } - if quiet, _ := config.Get(quietFlag); quiet == "true" { - stdout = ioutil.Discard + c.config = config + return nil +} + +func (c *CLI) configureOutput(cmd *cobra.Command, args []string) error { + if f, ok := c.Stdout.(*os.File); ok { + c.Stdout = colorable.NewColorable(f) + } + if f, ok := c.Stderr.(*os.File); ok { + c.Stderr = colorable.NewColorable(f) + } + if quiet, _ := c.config.get(quietFlag); quiet == "true" { + c.Stdout = ioutil.Discard } log.SetFlags(0) // No timestamps - log.SetOutput(stdout) - colorValue, _ := config.Get(colorFlag) + log.SetOutput(c.Stdout) + colorValue, _ := c.config.get(colorFlag) colorize := false switch colorValue { case "auto": - _, nocolor := os.LookupEnv("NO_COLOR") // https://no-color.org - colorize = !nocolor && isTerminal() + _, nocolor := c.Environment["NO_COLOR"] // https://no-color.org + colorize = !nocolor && c.isTerminal() case "always": colorize = true case "never": @@ -106,39 +192,328 @@ func configureOutput() error { return nil } -func init() { - rootCmd.PersistentFlags().StringVarP(&targetArg, targetFlag, "t", "local", "The name or URL of the recipient of this command") - rootCmd.PersistentFlags().StringVarP(&applicationArg, applicationFlag, "a", "", "The application to manage") - rootCmd.PersistentFlags().IntVarP(&waitSecsArg, waitFlag, "w", 0, "Number of seconds to wait for a service to become ready") - rootCmd.PersistentFlags().StringVarP(&colorArg, colorFlag, "c", "auto", "Whether to use colors in output. Can be \"auto\", \"never\" or \"always\"") - rootCmd.PersistentFlags().BoolVarP(&quietArg, quietFlag, "q", false, "Quiet mode. Only errors are printed.") - rootCmd.PersistentFlags().StringVarP(&apiKeyFileArg, apiKeyFileFlag, "k", "", "Path to API key used for deployment authentication") +func (c *CLI) configureFlags() { + flags := Flags{} + c.cmd.PersistentFlags().StringVarP(&flags.target, targetFlag, "t", "local", "The name or URL of the recipient of this command") + c.cmd.PersistentFlags().StringVarP(&flags.application, applicationFlag, "a", "", "The application to manage") + c.cmd.PersistentFlags().IntVarP(&flags.waitSecs, waitFlag, "w", 0, "Number of seconds to wait for a service to become ready") + c.cmd.PersistentFlags().StringVarP(&flags.color, colorFlag, "c", "auto", "Whether to use colors in output.") + c.cmd.PersistentFlags().BoolVarP(&flags.quiet, quietFlag, "q", false, "Quiet mode. Only errors will be printed") + c.cmd.PersistentFlags().StringVarP(&flags.apiKeyFile, apiKeyFileFlag, "k", "", "Path to API key used for cloud authentication") + c.flags = &flags +} + +func (c *CLI) configureCommands() { + rootCmd := c.cmd + authCmd := newAuthCmd() + certCmd := newCertCmd(c, false) + configCmd := newConfigCmd() + documentCmd := newDocumentCmd(c) + prodCmd := newProdCmd() + statusCmd := newStatusCmd(c) + certCmd.AddCommand(newCertAddCmd(c)) // auth cert add + authCmd.AddCommand(certCmd) // auth cert + authCmd.AddCommand(newAPIKeyCmd(c, false)) // auth api-key + authCmd.AddCommand(newLoginCmd(c)) // auth login + authCmd.AddCommand(newLogoutCmd(c)) // auth logout + rootCmd.AddCommand(authCmd) // auth + rootCmd.AddCommand(newCertCmd(c, true)) // cert TODO: Remove this after 2022-06-01 + rootCmd.AddCommand(newAPIKeyCmd(c, true)) // api-key TODO: Remove this after 2022-06-01 + rootCmd.AddCommand(newCloneCmd(c)) // clone + configCmd.AddCommand(newConfigGetCmd(c)) // config get + configCmd.AddCommand(newConfigSetCmd(c)) // config set + rootCmd.AddCommand(configCmd) // config + rootCmd.AddCommand(newCurlCmd(c)) // curl + rootCmd.AddCommand(newDeployCmd(c)) // deploy + rootCmd.AddCommand(newPrepareCmd(c)) // prepare + rootCmd.AddCommand(newActivateCmd(c)) // activate + documentCmd.AddCommand(newDocumentPutCmd(c)) // document put + documentCmd.AddCommand(newDocumentUpdateCmd(c)) // document update + documentCmd.AddCommand(newDocumentRemoveCmd(c)) // document remove + documentCmd.AddCommand(newDocumentGetCmd(c)) // document get + rootCmd.AddCommand(documentCmd) // document + rootCmd.AddCommand(newLogCmd(c)) // log + rootCmd.AddCommand(newManCmd(c)) // man + prodCmd.AddCommand(newProdInitCmd(c)) // prod init + prodCmd.AddCommand(newProdSubmitCmd(c)) // prod submit + rootCmd.AddCommand(prodCmd) // prod + rootCmd.AddCommand(newQueryCmd(c)) // query + statusCmd.AddCommand(newStatusQueryCmd(c)) // status query + statusCmd.AddCommand(newStatusDocumentCmd(c)) // status document + statusCmd.AddCommand(newStatusDeployCmd(c)) // status deploy + rootCmd.AddCommand(statusCmd) // status + rootCmd.AddCommand(newTestCmd(c)) // test + rootCmd.AddCommand(newVersionCmd(c)) // version +} + +func (c *CLI) printErrHint(err error, hints ...string) { + fmt.Fprintln(c.Stderr, color.RedString("Error:"), err) + for _, hint := range hints { + fmt.Fprintln(c.Stderr, color.CyanString("Hint:"), hint) + } +} - bindFlagToConfig(targetFlag, rootCmd) - bindFlagToConfig(applicationFlag, rootCmd) - bindFlagToConfig(waitFlag, rootCmd) - bindFlagToConfig(colorFlag, rootCmd) - bindFlagToConfig(quietFlag, rootCmd) - bindFlagToConfig(apiKeyFileFlag, rootCmd) +func (c *CLI) printSuccess(msg ...interface{}) { + log.Print(color.GreenString("Success: "), fmt.Sprint(msg...)) +} - bindEnvToConfig(apiKeyFlag, "VESPA_CLI_API_KEY") - bindEnvToConfig(apiKeyFileFlag, "VESPA_CLI_API_KEY_FILE") +func (c *CLI) printWarning(msg string, hints ...string) { + fmt.Fprintln(c.Stderr, color.YellowString("Warning:"), msg) + for _, hint := range hints { + fmt.Fprintln(c.Stderr, color.CyanString("Hint:"), hint) + } } -// errHint creates a new CLI error, with optional hints that will be printed after the error -func errHint(err error, hints ...string) ErrCLI { return ErrCLI{Status: 1, hints: hints, error: err} } +// target creates a target according the configuration of this CLI and given opts. +func (c *CLI) target(opts targetOptions) (vespa.Target, error) { + target, err := c.createTarget(opts) + if err != nil { + return nil, err + } + if !c.isCloudCI() { // Vespa Cloud always runs an up-to-date version + if err := target.CheckVersion(c.version); err != nil { + c.printErrHint(err, "This is not a fatal error, but this version may not work as expected", "Try 'vespa version' to check for a new version") + } + } + return target, nil +} + +func (c *CLI) createTarget(opts targetOptions) (vespa.Target, error) { + targetType, err := c.config.targetType() + if err != nil { + return nil, err + } + if strings.HasPrefix(targetType, "http") { + return vespa.CustomTarget(c.httpClient, targetType), nil + } + switch targetType { + case vespa.TargetLocal: + return vespa.LocalTarget(c.httpClient), nil + case vespa.TargetCloud, vespa.TargetHosted: + return c.createCloudTarget(targetType, opts) + } + return nil, errHint(fmt.Errorf("invalid target: %s", targetType), "Valid targets are 'local', 'cloud', 'hosted' or an URL") +} -// Execute executes command and prints any errors. -func Execute() error { - err := rootCmd.Execute() +func (c *CLI) createCloudTarget(targetType string, opts targetOptions) (vespa.Target, error) { + system, err := c.system(targetType) + if err != nil { + return nil, err + } + deployment, err := c.config.deploymentIn(opts.zone, system) + if err != nil { + return nil, err + } + endpoints, err := c.endpointsFromEnv() + if err != nil { + return nil, err + } + var ( + apiKey []byte + authConfigPath string + apiTLSOptions vespa.TLSOptions + deploymentTLSOptions vespa.TLSOptions + ) + switch targetType { + case vespa.TargetCloud: + if c.config.useAPIKey(c, system, deployment.Application.Tenant) { + apiKey, err = c.config.readAPIKey(deployment.Application.Tenant) + if err != nil { + return nil, err + } + } + authConfigPath = c.config.authConfigPath() + deploymentTLSOptions = vespa.TLSOptions{} + if !opts.noCertificate { + kp, err := c.config.x509KeyPair(deployment.Application) + if err != nil { + return nil, errHint(err, "Deployment to cloud requires a certificate. Try 'vespa auth cert'") + } + deploymentTLSOptions = vespa.TLSOptions{ + KeyPair: kp.KeyPair, + CertificateFile: kp.CertificateFile, + PrivateKeyFile: kp.PrivateKeyFile, + } + } + case vespa.TargetHosted: + kp, err := athenzKeyPair() + if err != nil { + return nil, err + } + apiTLSOptions = vespa.TLSOptions{ + KeyPair: kp.KeyPair, + CertificateFile: kp.CertificateFile, + PrivateKeyFile: kp.PrivateKeyFile, + } + deploymentTLSOptions = apiTLSOptions + default: + return nil, fmt.Errorf("invalid cloud target: %s", targetType) + } + apiOptions := vespa.APIOptions{ + System: system, + TLSOptions: apiTLSOptions, + APIKey: apiKey, + AuthConfigPath: authConfigPath, + } + deploymentOptions := vespa.CloudDeploymentOptions{ + Deployment: deployment, + TLSOptions: deploymentTLSOptions, + ClusterURLs: endpoints, + } + logLevel := opts.logLevel + if logLevel == "" { + logLevel = "info" + } + logOptions := vespa.LogOptions{ + Writer: c.Stdout, + Level: vespa.LogLevel(logLevel), + } + return vespa.CloudTarget(c.httpClient, apiOptions, deploymentOptions, logOptions) +} + +// system returns the appropiate system for the target configured in this CLI. +func (c *CLI) system(targetType string) (vespa.System, error) { + name := c.Environment["VESPA_CLI_CLOUD_SYSTEM"] + if name != "" { + return vespa.GetSystem(name) + } + switch targetType { + case vespa.TargetHosted: + return vespa.MainSystem, nil + case vespa.TargetCloud: + return vespa.PublicSystem, nil + } + return vespa.System{}, fmt.Errorf("no default system found for %s target", targetType) +} + +// service returns the service identified by given name and optionally cluster. This function blocks according to the +// wait period configured in this CLI. The parameter sessionOrRunID specifies either the session ID (local target) or +// run ID (cloud target) to wait for. +func (c *CLI) service(name string, sessionOrRunID int64, cluster string) (*vespa.Service, error) { + t, err := c.target(targetOptions{}) + if err != nil { + return nil, err + } + timeout := time.Duration(c.flags.waitSecs) * time.Second + if timeout > 0 { + log.Printf("Waiting up to %s %s for %s service to become available ...", color.CyanString(strconv.Itoa(c.flags.waitSecs)), color.CyanString("seconds"), color.CyanString(name)) + } + s, err := t.Service(name, timeout, sessionOrRunID, cluster) + if err != nil { + return nil, fmt.Errorf("service '%s' is unavailable: %w", name, err) + } + return s, nil +} + +func (c *CLI) createDeploymentOptions(pkg vespa.ApplicationPackage, target vespa.Target) vespa.DeploymentOptions { + return vespa.DeploymentOptions{ + ApplicationPackage: pkg, + Target: target, + Timeout: time.Duration(c.flags.waitSecs) * time.Second, + HTTPClient: c.httpClient, + } +} + +// isCI returns true if running inside a continuous integration environment. +func (c *CLI) isCI() bool { + _, ok := c.Environment["CI"] + return ok +} + +// isCloudCI returns true if running inside a Vespa Cloud deployment job. +func (c *CLI) isCloudCI() bool { + _, ok := c.Environment["VESPA_CLI_CLOUD_CI"] + return ok +} + +func (c *CLI) endpointsFromEnv() (map[string]string, error) { + endpointsString := c.Environment["VESPA_CLI_ENDPOINTS"] + if endpointsString == "" { + return nil, nil + } + var endpoints endpoints + urlsByCluster := make(map[string]string) + if err := json.Unmarshal([]byte(endpointsString), &endpoints); err != nil { + return nil, fmt.Errorf("endpoints must be valid json: %w", err) + } + if len(endpoints.Endpoints) == 0 { + return nil, fmt.Errorf("endpoints must be non-empty") + } + for _, endpoint := range endpoints.Endpoints { + urlsByCluster[endpoint.Cluster] = endpoint.URL + } + return urlsByCluster, nil +} + +// Run executes the CLI with given args. If args is nil, it defaults to os.Args[1:]. +func (c *CLI) Run(args ...string) error { + c.cmd.SetArgs(args) + err := c.cmd.Execute() if err != nil { if cliErr, ok := err.(ErrCLI); ok { if !cliErr.quiet { - printErrHint(cliErr, cliErr.hints...) + c.printErrHint(cliErr, cliErr.hints...) } } else { - printErrHint(err) + c.printErrHint(err) } } return err } + +type endpoints struct { + Endpoints []endpoint `json:"endpoints"` +} + +type endpoint struct { + Cluster string `json:"cluster"` + URL string `json:"url"` +} + +func isTerminal(w io.Writer) bool { + if f, ok := w.(*os.File); ok { + return isatty.IsTerminal(f.Fd()) + } + return false +} + +func athenzPath(filename string) (string, error) { + userHome, err := os.UserHomeDir() + if err != nil { + return "", err + } + return filepath.Join(userHome, ".athenz", filename), nil +} + +func athenzKeyPair() (KeyPair, error) { + certFile, err := athenzPath("cert") + if err != nil { + return KeyPair{}, err + } + keyFile, err := athenzPath("key") + if err != nil { + return KeyPair{}, err + } + kp, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return KeyPair{}, err + } + cert, err := x509.ParseCertificate(kp.Certificate[0]) + if err != nil { + return KeyPair{}, err + } + now := time.Now() + expiredAt := cert.NotAfter + if expiredAt.Before(now) { + delta := now.Sub(expiredAt).Truncate(time.Second) + return KeyPair{}, errHint(fmt.Errorf("certificate %s expired at %s (%s ago)", certFile, cert.NotAfter, delta), "Try renewing certificate with 'athenz-user-cert'") + } + return KeyPair{KeyPair: kp, CertificateFile: certFile, PrivateKeyFile: keyFile}, nil +} + +func applicationSource(args []string) string { + if len(args) > 0 { + return args[0] + } + return "." +} diff --git a/client/go/cmd/status.go b/client/go/cmd/status.go index 93316b7b6de..1eacff56354 100644 --- a/client/go/cmd/status.go +++ b/client/go/cmd/status.go @@ -5,61 +5,93 @@ package cmd import ( + "fmt" + "log" + "strconv" + "time" + + "github.com/fatih/color" "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/vespa" ) -func init() { - rootCmd.AddCommand(statusCmd) - statusCmd.AddCommand(statusQueryCmd) - statusCmd.AddCommand(statusDocumentCmd) - statusCmd.AddCommand(statusDeployCmd) +func newStatusCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "status", + Short: "Verify that a service is ready to use (query by default)", + Example: `$ vespa status query`, + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return printServiceStatus(cli, vespa.QueryService) + }, + } } -var statusCmd = &cobra.Command{ - Use: "status", - Short: "Verify that a service is ready to use (query by default)", - Example: `$ vespa status query`, - DisableAutoGenTag: true, - SilenceUsage: true, - Args: cobra.MaximumNArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - return waitForService(vespa.QueryService, 0) - }, +func newStatusQueryCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "query", + Short: "Verify that the query service is ready to use (default)", + Example: `$ vespa status query`, + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + return printServiceStatus(cli, vespa.QueryService) + }, + } } -var statusQueryCmd = &cobra.Command{ - Use: "query", - Short: "Verify that the query service is ready to use (default)", - Example: `$ vespa status query`, - DisableAutoGenTag: true, - SilenceUsage: true, - Args: cobra.ExactArgs(0), - RunE: func(cmd *cobra.Command, args []string) error { - return waitForService(vespa.QueryService, 0) - }, +func newStatusDocumentCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "document", + Short: "Verify that the document service is ready to use", + Example: `$ vespa status document`, + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + return printServiceStatus(cli, vespa.DocumentService) + }, + } } -var statusDocumentCmd = &cobra.Command{ - Use: "document", - Short: "Verify that the document service is ready to use", - Example: `$ vespa status document`, - DisableAutoGenTag: true, - SilenceUsage: true, - Args: cobra.ExactArgs(0), - RunE: func(cmd *cobra.Command, args []string) error { - return waitForService(vespa.DocumentService, 0) - }, +func newStatusDeployCmd(cli *CLI) *cobra.Command { + return &cobra.Command{ + Use: "deploy", + Short: "Verify that the deploy service is ready to use", + Example: `$ vespa status deploy`, + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + return printServiceStatus(cli, vespa.DeployService) + }, + } } -var statusDeployCmd = &cobra.Command{ - Use: "deploy", - Short: "Verify that the deploy service is ready to use", - Example: `$ vespa status deploy`, - DisableAutoGenTag: true, - SilenceUsage: true, - Args: cobra.ExactArgs(0), - RunE: func(cmd *cobra.Command, args []string) error { - return waitForService(vespa.DeployService, 0) - }, +func printServiceStatus(cli *CLI, name string) error { + t, err := cli.target(targetOptions{}) + if err != nil { + return err + } + timeout := time.Duration(cli.flags.waitSecs) * time.Second + if timeout > 0 { + log.Printf("Waiting up to %s %s for service to become ready ...", color.CyanString(strconv.Itoa(cli.flags.waitSecs)), color.CyanString("seconds")) + } + s, err := t.Service(name, timeout, 0, "") + if err != nil { + return err + } + status, err := s.Wait(timeout) + if status/100 == 2 { + log.Print(s.Description(), " at ", color.CyanString(s.BaseURL), " is ", color.GreenString("ready")) + } else { + if err == nil { + err = fmt.Errorf("status %d", status) + } + return fmt.Errorf("%s at %s is %s: %w", s.Description(), color.CyanString(s.BaseURL), color.RedString("not ready"), err) + } + return nil } diff --git a/client/go/cmd/status_test.go b/client/go/cmd/status_test.go index fe7228697c7..7dc2aeace84 100644 --- a/client/go/cmd/status_test.go +++ b/client/go/cmd/status_test.go @@ -45,33 +45,47 @@ func TestStatusErrorResponse(t *testing.T) { func assertDeployStatus(target string, args []string, t *testing.T) { client := &mock.HTTPClient{} + cli, stdout, _ := newTestCLI(t) + cli.httpClient = client + statusArgs := []string{"status", "deploy"} + assert.Nil(t, cli.Run(append(statusArgs, args...)...)) assert.Equal(t, "Deploy API at "+target+" is ready\n", - executeCommand(t, client, []string{"status", "deploy"}, args), + stdout.String(), "vespa status config-server") assert.Equal(t, target+"/status.html", client.LastRequest.URL.String()) } func assertQueryStatus(target string, args []string, t *testing.T) { client := &mock.HTTPClient{} + cli, stdout, _ := newTestCLI(t) + cli.httpClient = client + statusArgs := []string{"status", "query"} + assert.Nil(t, cli.Run(append(statusArgs, args...)...)) assert.Equal(t, "Container (query API) at "+target+" is ready\n", - executeCommand(t, client, []string{"status", "query"}, args), + stdout.String(), "vespa status container") assert.Equal(t, target+"/ApplicationStatus", client.LastRequest.URL.String()) + statusArgs = []string{"status"} + stdout.Reset() + assert.Nil(t, cli.Run(append(statusArgs, args...)...)) assert.Equal(t, "Container (query API) at "+target+" is ready\n", - executeCommand(t, client, []string{"status"}, args), + stdout.String(), "vespa status (the default)") assert.Equal(t, target+"/ApplicationStatus", client.LastRequest.URL.String()) } func assertDocumentStatus(target string, args []string, t *testing.T) { client := &mock.HTTPClient{} + cli, stdout, _ := newTestCLI(t) + cli.httpClient = client + assert.Nil(t, cli.Run("status", "document")) assert.Equal(t, "Container (document API) at "+target+" is ready\n", - executeCommand(t, client, []string{"status", "document"}, args), + stdout.String(), "vespa status container") assert.Equal(t, target+"/ApplicationStatus", client.LastRequest.URL.String()) } @@ -79,11 +93,11 @@ func assertDocumentStatus(target string, args []string, t *testing.T) { func assertQueryStatusError(target string, args []string, t *testing.T) { client := &mock.HTTPClient{} client.NextStatus(500) - cmd := []string{"status", "container"} - cmd = append(cmd, args...) - _, outErr := execute(command{args: cmd}, t, client) + cli, _, stderr := newTestCLI(t) + cli.httpClient = client + assert.NotNil(t, cli.Run("status", "container")) assert.Equal(t, "Error: Container (query API) at "+target+" is not ready: status 500\n", - outErr, + stderr.String(), "vespa status container") } diff --git a/client/go/cmd/test.go b/client/go/cmd/test.go index 56ea9277842..69dc61e7e0e 100644 --- a/client/go/cmd/test.go +++ b/client/go/cmd/test.go @@ -25,51 +25,51 @@ import ( "github.com/vespa-engine/vespa/client/go/vespa" ) -func init() { - rootCmd.AddCommand(testCmd) - testCmd.PersistentFlags().StringVarP(&zoneArg, zoneFlag, "z", "", "The zone to use for deployment. This defaults to a dev zone") -} - -var testCmd = &cobra.Command{ - Use: "test test-directory-or-file", - Short: "Run a test suite, or a single test", - Long: `Run a test suite, or a single test +func newTestCmd(cli *CLI) *cobra.Command { + var zoneArg string + testCmd := &cobra.Command{ + Use: "test test-directory-or-file", + Short: "Run a test suite, or a single test", + Long: `Run a test suite, or a single test Runs all JSON test files in the specified directory, or the single JSON test file specified. See https://docs.vespa.ai/en/reference/testing.html for details.`, - Example: `$ vespa test src/test/application/tests/system-test + Example: `$ vespa test src/test/application/tests/system-test $ vespa test src/test/application/tests/system-test/feed-and-query.json`, - Args: cobra.ExactArgs(1), - DisableAutoGenTag: true, - SilenceUsage: true, - RunE: func(cmd *cobra.Command, args []string) error { - count, failed, err := runTests(args[0], false) - if err != nil { - return err - } - if len(failed) != 0 { - plural := "s" - if count == 1 { - plural = "" - } - fmt.Fprintf(stdout, "\n%s %d of %d test%s failed:\n", color.RedString("Failure:"), len(failed), count, plural) - for _, test := range failed { - fmt.Fprintln(stdout, test) + Args: cobra.ExactArgs(1), + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + count, failed, err := runTests(cli, zoneArg, args[0], false) + if err != nil { + return err } - return ErrCLI{Status: 3, error: fmt.Errorf("tests failed"), quiet: true} - } else { - plural := "s" - if count == 1 { - plural = "" + if len(failed) != 0 { + plural := "s" + if count == 1 { + plural = "" + } + fmt.Fprintf(cli.Stdout, "\n%s %d of %d test%s failed:\n", color.RedString("Failure:"), len(failed), count, plural) + for _, test := range failed { + fmt.Fprintln(cli.Stdout, test) + } + return ErrCLI{Status: 3, error: fmt.Errorf("tests failed"), quiet: true} + } else { + plural := "s" + if count == 1 { + plural = "" + } + fmt.Fprintf(cli.Stdout, "\n%s %d test%s OK\n", color.GreenString("Success:"), count, plural) + return nil } - fmt.Fprintf(stdout, "\n%s %d test%s OK\n", color.GreenString("Success:"), count, plural) - return nil - } - }, + }, + } + testCmd.PersistentFlags().StringVarP(&zoneArg, "zone", "z", "", "The zone to use for deployment. This defaults to a dev zone") + return testCmd } -func runTests(rootPath string, dryRun bool) (int, []string, error) { +func runTests(cli *CLI, zone, rootPath string, dryRun bool) (int, []string, error) { count := 0 failed := make([]string, 0) if stat, err := os.Stat(rootPath); err != nil { @@ -79,13 +79,13 @@ func runTests(rootPath string, dryRun bool) (int, []string, error) { if err != nil { return 0, nil, errHint(err, "See https://docs.vespa.ai/en/reference/testing") } - context := testContext{testsPath: rootPath, dryRun: dryRun} + context := testContext{testsPath: rootPath, dryRun: dryRun, cli: cli, zone: zone} previousFailed := false for _, test := range tests { if !test.IsDir() && filepath.Ext(test.Name()) == ".json" { testPath := filepath.Join(rootPath, test.Name()) if previousFailed { - fmt.Fprintln(stdout, "") + fmt.Fprintln(cli.Stdout, "") previousFailed = false } failure, err := runTest(testPath, context) @@ -100,7 +100,7 @@ func runTests(rootPath string, dryRun bool) (int, []string, error) { } } } else if strings.HasSuffix(stat.Name(), ".json") { - failure, err := runTest(rootPath, testContext{testsPath: filepath.Dir(rootPath), dryRun: dryRun}) + failure, err := runTest(rootPath, testContext{testsPath: filepath.Dir(rootPath), dryRun: dryRun, cli: cli}) if err != nil { return 0, nil, err } @@ -131,17 +131,17 @@ func runTest(testPath string, context testContext) (string, error) { testName = filepath.Base(testPath) } if !context.dryRun { - fmt.Fprintf(stdout, "%s:", testName) + fmt.Fprintf(context.cli.Stdout, "%s:", testName) } defaultParameters, err := getParameters(test.Defaults.ParametersRaw, filepath.Dir(testPath)) if err != nil { - fmt.Fprintln(stderr) + fmt.Fprintln(context.cli.Stderr) return "", errHint(fmt.Errorf("invalid default parameters for %s: %w", testName, err), "See https://docs.vespa.ai/en/reference/testing") } if len(test.Steps) == 0 { - fmt.Fprintln(stderr) + fmt.Fprintln(context.cli.Stderr) return "", errHint(fmt.Errorf("a test must have at least one step, but none were found in %s", testPath), "See https://docs.vespa.ai/en/reference/testing") } for i, step := range test.Steps { @@ -151,22 +151,22 @@ func runTest(testPath string, context testContext) (string, error) { } failure, longFailure, err := verify(step, test.Defaults.Cluster, defaultParameters, context) if err != nil { - fmt.Fprintln(stderr) + fmt.Fprintln(context.cli.Stderr) return "", errHint(fmt.Errorf("error in %s: %w", stepName, err), "See https://docs.vespa.ai/en/reference/testing") } if !context.dryRun { if failure != "" { - fmt.Fprintf(stdout, " %s\n%s:\n%s\n", color.RedString("failed"), stepName, longFailure) + fmt.Fprintf(context.cli.Stdout, " %s\n%s:\n%s\n", color.RedString("failed"), stepName, longFailure) return fmt.Sprintf("%s: %s: %s", testName, stepName, failure), nil } if i == 0 { - fmt.Fprintf(stdout, " ") + fmt.Fprintf(context.cli.Stdout, " ") } - fmt.Fprint(stdout, ".") + fmt.Fprint(context.cli.Stdout, ".") } } if !context.dryRun { - fmt.Fprintln(stdout, color.GreenString(" OK")) + fmt.Fprintln(context.cli.Stdout, color.GreenString(" OK")) } return "", nil } @@ -265,8 +265,8 @@ func verify(step step, defaultCluster string, defaultParameters map[string]strin var response *http.Response if externalEndpoint { - util.ActiveHttpClient.UseCertificate([]tls.Certificate{}) - response, err = util.ActiveHttpClient.Do(request, 60*time.Second) + context.cli.httpClient.UseCertificate([]tls.Certificate{}) + response, err = context.cli.httpClient.Do(request, 60*time.Second) } else { response, err = service.Do(request, 600*time.Second) // Vespa should provide a response within the given request timeout } @@ -384,8 +384,8 @@ func compare(expected interface{}, actual interface{}, path string) (string, str expectedJson, _ := json.Marshal(expected) actualJson, _ := json.Marshal(actual) return fmt.Sprintf("Unexpected %s at %s", mismatched, color.CyanString(path)), - fmt.Sprintf("%s", color.CyanString(string(expectedJson))), - fmt.Sprintf("%s", color.RedString(string(actualJson))), + color.CyanString(string(expectedJson)), + color.RedString(string(actualJson)), nil } return "", "", "", nil @@ -470,6 +470,8 @@ type response struct { } type testContext struct { + cli *CLI + zone string lazyTarget vespa.Target testsPath string dryRun bool @@ -477,7 +479,7 @@ type testContext struct { func (t *testContext) target() (vespa.Target, error) { if t.lazyTarget == nil { - target, err := getTarget() + target, err := t.cli.target(targetOptions{zone: t.zone}) if err != nil { return nil, err } diff --git a/client/go/cmd/test_test.go b/client/go/cmd/test_test.go index 1f7d0cff7b2..c3538c7cb1a 100644 --- a/client/go/cmd/test_test.go +++ b/client/go/cmd/test_test.go @@ -8,7 +8,6 @@ import ( "io/ioutil" "net/http" "net/url" - "os" "path/filepath" "strings" "testing" @@ -30,7 +29,9 @@ func TestSuite(t *testing.T) { } expectedBytes, _ := ioutil.ReadFile("testdata/tests/expected-suite.out") - outBytes, errBytes := execute(command{args: []string{"test", "testdata/tests/system-test"}}, t, client) + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = client + assert.NotNil(t, cli.Run("test", "testdata/tests/system-test")) baseUrl := "http://127.0.0.1:8080" urlWithQuery := baseUrl + "/search/?presentation.timing=true&query=artist%3A+foo&timeout=3.4s" @@ -41,47 +42,53 @@ func TestSuite(t *testing.T) { requests = append(requests, createSearchRequest(baseUrl+"/search/")) } assertRequests(requests, client, t) - assert.Equal(t, string(expectedBytes), outBytes) - assert.Equal(t, "", errBytes) + assert.Equal(t, string(expectedBytes), stdout.String()) + assert.Equal(t, "", stderr.String()) } func TestIllegalFileReference(t *testing.T) { client := &mock.HTTPClient{} client.NextStatus(200) client.NextStatus(200) - _, errBytes := execute(command{args: []string{"test", "testdata/tests/production-test/illegal-reference.json"}}, t, client) + cli, _, stderr := newTestCLI(t) + cli.httpClient = client + assert.NotNil(t, cli.Run("test", "testdata/tests/production-test/illegal-reference.json")) assertRequests([]*http.Request{createRequest("GET", "https://domain.tld", "{}")}, client, t) - assert.Equal(t, "\nError: error in Step 2: path may not point outside src/test/application, but 'foo/../../../../this-is-not-ok.json' does\nHint: See https://docs.vespa.ai/en/reference/testing\n", errBytes) + assert.Equal(t, "\nError: error in Step 2: path may not point outside src/test/application, but 'foo/../../../../this-is-not-ok.json' does\nHint: See https://docs.vespa.ai/en/reference/testing\n", stderr.String()) } func TestIllegalRequestUri(t *testing.T) { client := &mock.HTTPClient{} client.NextStatus(200) client.NextStatus(200) - _, errBytes := execute(command{args: []string{"test", "testdata/tests/production-test/illegal-uri.json"}}, t, client) + cli, _, stderr := newTestCLI(t) + cli.httpClient = client + assert.NotNil(t, cli.Run("test", "testdata/tests/production-test/illegal-uri.json")) assertRequests([]*http.Request{createRequest("GET", "https://domain.tld/my-api", "")}, client, t) - assert.Equal(t, "\nError: error in Step 2: production tests may not specify requests against Vespa endpoints\nHint: See https://docs.vespa.ai/en/reference/testing\n", errBytes) + assert.Equal(t, "\nError: error in Step 2: production tests may not specify requests against Vespa endpoints\nHint: See https://docs.vespa.ai/en/reference/testing\n", stderr.String()) } func TestProductionTest(t *testing.T) { client := &mock.HTTPClient{} client.NextStatus(200) - outBytes, errBytes := execute(command{args: []string{"test", "testdata/tests/production-test/external.json"}}, t, client) - assert.Equal(t, "external.json: . OK\n\nSuccess: 1 test OK\n", outBytes) - assert.Equal(t, "", errBytes) + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = client + assert.Nil(t, cli.Run("test", "testdata/tests/production-test/external.json")) + assert.Equal(t, "external.json: . OK\n\nSuccess: 1 test OK\n", stdout.String()) + assert.Equal(t, "", stderr.String()) assertRequests([]*http.Request{createRequest("GET", "https://my.service:123/path?query=wohoo", "")}, client, t) } func TestTestWithoutAssertions(t *testing.T) { - client := &mock.HTTPClient{} - _, errBytes := execute(command{args: []string{"test", "testdata/tests/system-test/foo/query.json"}}, t, client) - assert.Equal(t, "\nError: a test must have at least one step, but none were found in testdata/tests/system-test/foo/query.json\nHint: See https://docs.vespa.ai/en/reference/testing\n", errBytes) + cli, _, stderr := newTestCLI(t) + assert.NotNil(t, cli.Run("test", "testdata/tests/system-test/foo/query.json")) + assert.Equal(t, "\nError: a test must have at least one step, but none were found in testdata/tests/system-test/foo/query.json\nHint: See https://docs.vespa.ai/en/reference/testing\n", stderr.String()) } func TestSuiteWithoutTests(t *testing.T) { - client := &mock.HTTPClient{} - _, errBytes := execute(command{args: []string{"test", "testdata/tests/staging-test"}}, t, client) - assert.Equal(t, "Error: failed to find any tests at testdata/tests/staging-test\nHint: See https://docs.vespa.ai/en/reference/testing\n", errBytes) + cli, _, stderr := newTestCLI(t) + assert.NotNil(t, cli.Run("test", "testdata/tests/staging-test")) + assert.Equal(t, "Error: failed to find any tests at testdata/tests/staging-test\nHint: See https://docs.vespa.ai/en/reference/testing\n", stderr.String()) } func TestSingleTest(t *testing.T) { @@ -91,11 +98,13 @@ func TestSingleTest(t *testing.T) { client.NextStatus(200) client.NextResponse(200, string(searchResponse)) client.NextResponse(200, string(searchResponse)) + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = client expectedBytes, _ := ioutil.ReadFile("testdata/tests/expected.out") - outBytes, errBytes := execute(command{args: []string{"test", "testdata/tests/system-test/test.json"}}, t, client) - assert.Equal(t, string(expectedBytes), outBytes) - assert.Equal(t, "", errBytes) + assert.Nil(t, cli.Run("test", "testdata/tests/system-test/test.json")) + assert.Equal(t, string(expectedBytes), stdout.String()) + assert.Equal(t, "", stderr.String()) baseUrl := "http://127.0.0.1:8080" rawUrl := baseUrl + "/search/?presentation.timing=true&query=artist%3A+foo&timeout=3.4s" @@ -105,34 +114,36 @@ func TestSingleTest(t *testing.T) { func TestSingleTestWithCloudAndEndpoints(t *testing.T) { apiKey, err := vespa.CreateAPIKey() require.Nil(t, err) - cmd := command{ - args: []string{"test", "testdata/tests/system-test/test.json", "-t", "cloud", "-a", "t.a.i"}, - env: map[string]string{"VESPA_CLI_API_KEY": string(apiKey)}, - } - cmd.homeDir = filepath.Join(t.TempDir(), ".vespa") - os.MkdirAll(cmd.homeDir, 0700) - keyFile := filepath.Join(cmd.homeDir, "key") - certFile := filepath.Join(cmd.homeDir, "cert") - - os.Setenv("VESPA_CLI_DATA_PLANE_KEY_FILE", keyFile) - os.Setenv("VESPA_CLI_DATA_PLANE_CERT_FILE", certFile) - os.Setenv("VESPA_CLI_ENDPOINTS", "{\"endpoints\":[{\"cluster\":\"container\",\"url\":\"https://url\"}]}") - - kp, _ := vespa.CreateKeyPair() - ioutil.WriteFile(keyFile, kp.PrivateKey, 0600) - ioutil.WriteFile(certFile, kp.Certificate, 0600) + certDir := filepath.Join(t.TempDir()) + keyFile := filepath.Join(certDir, "key") + certFile := filepath.Join(certDir, "cert") + kp, err := vespa.CreateKeyPair() + require.Nil(t, err) + require.Nil(t, ioutil.WriteFile(keyFile, kp.PrivateKey, 0600)) + require.Nil(t, ioutil.WriteFile(certFile, kp.Certificate, 0600)) client := &mock.HTTPClient{} - searchResponse, _ := ioutil.ReadFile("testdata/tests/response.json") + cli, stdout, stderr := newTestCLI( + t, + "VESPA_CLI_API_KEY="+string(apiKey), + "VESPA_CLI_DATA_PLANE_KEY_FILE="+keyFile, + "VESPA_CLI_DATA_PLANE_CERT_FILE="+certFile, + "VESPA_CLI_ENDPOINTS={\"endpoints\":[{\"cluster\":\"container\",\"url\":\"https://url\"}]}", + ) + cli.httpClient = client + + searchResponse, err := ioutil.ReadFile("testdata/tests/response.json") + require.Nil(t, err) client.NextStatus(200) client.NextStatus(200) client.NextResponse(200, string(searchResponse)) client.NextResponse(200, string(searchResponse)) - expectedBytes, _ := ioutil.ReadFile("testdata/tests/expected.out") - outBytes, errBytes := execute(cmd, t, client) - assert.Equal(t, string(expectedBytes), outBytes) - assert.Equal(t, "", errBytes) + assert.Nil(t, cli.Run("test", "testdata/tests/system-test/test.json", "-t", "cloud", "-a", "t.a.i")) + expectedBytes, err := ioutil.ReadFile("testdata/tests/expected.out") + require.Nil(t, err) + assert.Equal(t, "", stderr.String()) + assert.Equal(t, string(expectedBytes), stdout.String()) baseUrl := "https://url" rawUrl := baseUrl + "/search/?presentation.timing=true&query=artist%3A+foo&timeout=3.4s" diff --git a/client/go/cmd/testutil_test.go b/client/go/cmd/testutil_test.go new file mode 100644 index 00000000000..68f79187d3a --- /dev/null +++ b/client/go/cmd/testutil_test.go @@ -0,0 +1,49 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package cmd + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/spf13/viper" + "github.com/vespa-engine/vespa/client/go/mock" +) + +func newTestCLI(t *testing.T, envVars ...string) (*CLI, *bytes.Buffer, *bytes.Buffer) { + t.Cleanup(viper.Reset) + homeDir := filepath.Join(t.TempDir(), ".vespa") + cacheDir := filepath.Join(t.TempDir(), ".cache", "vespa") + env := []string{"VESPA_CLI_HOME=" + homeDir, "VESPA_CLI_CACHE_DIR=" + cacheDir} + env = append(env, envVars...) + var ( + stdout bytes.Buffer + stderr bytes.Buffer + ) + cli, err := New(&stdout, &stderr, env) + if err != nil { + t.Fatal(err) + } + cli.httpClient = &mock.HTTPClient{} + cli.exec = &mock.Exec{} + return cli, &stdout, &stderr +} + +func mockApplicationPackage(t *testing.T, java bool) string { + dir := t.TempDir() + appDir := filepath.Join(dir, "src", "main", "application") + if err := os.MkdirAll(appDir, 0755); err != nil { + t.Fatal(err) + } + servicesXML := filepath.Join(appDir, "services.xml") + if _, err := os.Create(servicesXML); err != nil { + t.Fatal(err) + } + if java { + if _, err := os.Create(filepath.Join(dir, "pom.xml")); err != nil { + t.Fatal(err) + } + } + return dir +} diff --git a/client/go/cmd/version.go b/client/go/cmd/version.go index 875fc0cc3c1..0a47039a37d 100644 --- a/client/go/cmd/version.go +++ b/client/go/cmd/version.go @@ -6,7 +6,6 @@ import ( "log" "net/http" "os" - "os/exec" "path/filepath" "runtime" "sort" @@ -16,61 +15,42 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/build" - "github.com/vespa-engine/vespa/client/go/util" "github.com/vespa-engine/vespa/client/go/version" ) -var skipVersionCheck bool - -var sp subprocess = &execSubprocess{} - -type subprocess interface { - pathOf(name string) (string, error) - outputOf(name string, args ...string) ([]byte, error) - isTerminal() bool -} - -type execSubprocess struct{} - -func (c *execSubprocess) pathOf(name string) (string, error) { return exec.LookPath(name) } -func (c *execSubprocess) isTerminal() bool { return isTerminal() } -func (c *execSubprocess) outputOf(name string, args ...string) ([]byte, error) { - return exec.Command(name, args...).Output() -} - -func init() { - rootCmd.AddCommand(versionCmd) - versionCmd.Flags().BoolVarP(&skipVersionCheck, "no-check", "n", false, "Do not check if a new version is available") -} - -var versionCmd = &cobra.Command{ - Use: "version", - Short: "Show current version and check for updates", - DisableAutoGenTag: true, - SilenceUsage: true, - Args: cobra.ExactArgs(0), - RunE: func(cmd *cobra.Command, args []string) error { - log.Printf("vespa version %s compiled with %v on %v/%v", build.Version, runtime.Version(), runtime.GOOS, runtime.GOARCH) - if !skipVersionCheck && sp.isTerminal() { - return checkVersion() - } - return nil - }, +func newVersionCmd(cli *CLI) *cobra.Command { + var skipVersionCheck bool + cmd := &cobra.Command{ + Use: "version", + Short: "Show current version and check for updates", + DisableAutoGenTag: true, + SilenceUsage: true, + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + log.Printf("vespa version %s compiled with %v on %v/%v", build.Version, runtime.Version(), runtime.GOOS, runtime.GOARCH) + if !skipVersionCheck && cli.isTerminal() { + return checkVersion(cli) + } + return nil + }, + } + cmd.Flags().BoolVarP(&skipVersionCheck, "no-check", "n", false, "Do not check if a new version is available") + return cmd } -func checkVersion() error { +func checkVersion(cli *CLI) error { current, err := version.Parse(build.Version) if err != nil { return err } - latest, err := latestRelease() + latest, err := latestRelease(cli) if err != nil { return err } if !current.Less(latest.Version) { return nil } - usingHomebrew := usingHomebrew() + usingHomebrew := usingHomebrew(cli) if usingHomebrew && latest.isRecent() { return nil // Allow some time for new release to appear in Homebrew repo } @@ -82,12 +62,12 @@ func checkVersion() error { return nil } -func latestRelease() (release, error) { +func latestRelease(cli *CLI) (release, error) { req, err := http.NewRequest("GET", "https://api.github.com/repos/vespa-engine/vespa/releases", nil) if err != nil { return release{}, err } - response, err := util.HttpDo(req, time.Minute, "GitHub") + response, err := cli.httpClient.Do(req, time.Minute) if err != nil { return release{}, err } @@ -118,12 +98,12 @@ func latestRelease() (release, error) { return releases[len(releases)-1], nil } -func usingHomebrew() bool { - selfPath, err := sp.pathOf("vespa") +func usingHomebrew(cli *CLI) bool { + selfPath, err := cli.exec.LookPath("vespa") if err != nil { return false } - brewPrefix, err := sp.outputOf("brew", "--prefix") + brewPrefix, err := cli.exec.Run("brew", "--prefix") if err != nil { return false } diff --git a/client/go/cmd/version_test.go b/client/go/cmd/version_test.go index 9c05b130e84..b78c66c9e7f 100644 --- a/client/go/cmd/version_test.go +++ b/client/go/cmd/version_test.go @@ -2,52 +2,44 @@ package cmd import ( - "fmt" "testing" "github.com/stretchr/testify/assert" "github.com/vespa-engine/vespa/client/go/mock" - "github.com/vespa-engine/vespa/client/go/util" ) func TestVersion(t *testing.T) { c := &mock.HTTPClient{} c.NextResponse(200, `[{"tag_name": "v1.2.3", "published_at": "2021-09-10T12:00:00Z"}]`) - util.ActiveHttpClient = c - sp = &mockSubprocess{} - out, _ := execute(command{args: []string{"version"}}, t, nil) - assert.Contains(t, out, "vespa version 0.0.0-devel compiled with") - assert.Contains(t, out, "New release available: 1.2.3\nhttps://github.com/vespa-engine/vespa/releases/tag/v1.2.3") + sp := &mock.Exec{} + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = c + cli.exec = sp + cli.isTerminal = func() bool { return true } + if err := cli.Run("version", "--color", "never"); err != nil { + t.Fatal(err) + } + assert.Equal(t, "", stderr.String()) + assert.Contains(t, stdout.String(), "vespa version 0.0.0-devel compiled with") + assert.Contains(t, stdout.String(), "New release available: 1.2.3\nhttps://github.com/vespa-engine/vespa/releases/tag/v1.2.3") } func TestVersionCheckHomebrew(t *testing.T) { c := &mock.HTTPClient{} c.NextResponse(200, `[{"tag_name": "v1.2.3", "published_at": "2021-09-10T12:00:00Z"}]`) - util.ActiveHttpClient = c - sp = &mockSubprocess{programPath: "/usr/local/bin/vespa", output: "/usr/local"} - out, _ := execute(command{args: []string{"version"}}, t, nil) - assert.Contains(t, out, "vespa version 0.0.0-devel compiled with") - assert.Contains(t, out, "New release available: 1.2.3\n"+ + sp := &mock.Exec{ProgramPath: "/usr/local/bin/vespa", CombinedOutput: "/usr/local"} + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = c + cli.exec = sp + cli.isTerminal = func() bool { return true } + if err := cli.Run("version", "--color", "never"); err != nil { + t.Fatal(err) + } + assert.Equal(t, "", stderr.String()) + assert.Contains(t, stdout.String(), "vespa version 0.0.0-devel compiled with") + assert.Contains(t, stdout.String(), "New release available: 1.2.3\n"+ "https://github.com/vespa-engine/vespa/releases/tag/v1.2.3\n"+ "\nUpgrade by running:\nbrew update && brew upgrade vespa-cli\n") } - -type mockSubprocess struct { - programPath string - output string -} - -func (c *mockSubprocess) pathOf(name string) (string, error) { - if c.programPath == "" { - return "", fmt.Errorf("no program path set in this mock") - } - return c.programPath, nil -} - -func (c *mockSubprocess) outputOf(name string, args ...string) ([]byte, error) { - return []byte(c.output), nil -} - -func (c *mockSubprocess) isTerminal() bool { return true } diff --git a/client/go/cmd/vespa/main.go b/client/go/cmd/vespa/main.go index f7ce064f3a5..5e49f633877 100644 --- a/client/go/cmd/vespa/main.go +++ b/client/go/cmd/vespa/main.go @@ -5,17 +5,29 @@ package main import ( + "fmt" "os" "github.com/vespa-engine/vespa/client/go/cmd" ) +func fatal(status int, err error) { + if err != nil { + fmt.Fprintln(os.Stderr, err) + } + os.Exit(status) +} + func main() { - if err := cmd.Execute(); err != nil { + cli, err := cmd.New(os.Stdout, os.Stderr, os.Environ()) + if err != nil { + fatal(1, err) + } + if err := cli.Run(); err != nil { if cliErr, ok := err.(cmd.ErrCLI); ok { - os.Exit(cliErr.Status) + fatal(cliErr.Status, nil) } else { - os.Exit(1) + fatal(1, nil) } } } diff --git a/client/go/mock/mock.go b/client/go/mock/http.go index f2fcf9c5960..f2fcf9c5960 100644 --- a/client/go/mock/mock.go +++ b/client/go/mock/http.go diff --git a/client/go/mock/process.go b/client/go/mock/process.go new file mode 100644 index 00000000000..5105bed8b0c --- /dev/null +++ b/client/go/mock/process.go @@ -0,0 +1,19 @@ +package mock + +import "fmt" + +type Exec struct { + ProgramPath string + CombinedOutput string +} + +func (c *Exec) LookPath(name string) (string, error) { + if c.ProgramPath == "" { + return "", fmt.Errorf("no program path set in this mock") + } + return c.ProgramPath, nil +} + +func (c *Exec) Run(name string, args ...string) ([]byte, error) { + return []byte(c.CombinedOutput), nil +} diff --git a/client/go/util/http.go b/client/go/util/http.go index bb70c3ec6db..ffdd3d1599b 100644 --- a/client/go/util/http.go +++ b/client/go/util/http.go @@ -1,7 +1,4 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// A HTTP wrapper which handles some errors and provides a way to replace the HTTP client by a mock. -// Author: bratseth - package util import ( @@ -13,45 +10,32 @@ import ( "github.com/vespa-engine/vespa/client/go/build" ) -// Set this to a mock HttpClient instead to unit test HTTP requests -var ActiveHttpClient = CreateClient(time.Second * 10) - -type HttpClient interface { +type HTTPClient interface { Do(request *http.Request, timeout time.Duration) (response *http.Response, error error) UseCertificate(certificate []tls.Certificate) } -type defaultHttpClient struct { +type defaultHTTPClient struct { client *http.Client } -func (c *defaultHttpClient) Do(request *http.Request, timeout time.Duration) (response *http.Response, error error) { +func (c *defaultHTTPClient) Do(request *http.Request, timeout time.Duration) (response *http.Response, error error) { if c.client.Timeout != timeout { // Set wanted timeout c.client.Timeout = timeout } + if request.Header == nil { + request.Header = make(http.Header) + } + request.Header.Set("User-Agent", fmt.Sprintf("Vespa CLI/%s", build.Version)) return c.client.Do(request) } -func (c *defaultHttpClient) UseCertificate(certificates []tls.Certificate) { +func (c *defaultHTTPClient) UseCertificate(certificates []tls.Certificate) { c.client.Transport = &http.Transport{TLSClientConfig: &tls.Config{ Certificates: certificates, }} } -func CreateClient(timeout time.Duration) HttpClient { - return &defaultHttpClient{ - client: &http.Client{Timeout: timeout}, - } -} - -func HttpDo(request *http.Request, timeout time.Duration, description string) (*http.Response, error) { - if request.Header == nil { - request.Header = make(http.Header) - } - request.Header.Set("User-Agent", fmt.Sprintf("Vespa CLI/%s", build.Version)) - response, err := ActiveHttpClient.Do(request, timeout) - if err != nil { - return nil, err - } - return response, nil +func CreateClient(timeout time.Duration) HTTPClient { + return &defaultHTTPClient{client: &http.Client{Timeout: timeout}} } diff --git a/client/go/util/http_test.go b/client/go/util/http_test.go deleted file mode 100644 index ccb809d198b..00000000000 --- a/client/go/util/http_test.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Basic testing of our HTTP client wrapper -// Author: bratseth - -package util - -import ( - "bytes" - "crypto/tls" - "io/ioutil" - "net/http" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -type mockHttpClient struct{} - -func (c mockHttpClient) Do(request *http.Request, timeout time.Duration) (response *http.Response, error error) { - var status int - var body string - if request.URL.String() == "http://host/okpath" { - status = 200 - body = "OK body" - } else { - status = 500 - body = "Unexpected url body" - } - - return &http.Response{ - StatusCode: status, - Header: make(http.Header), - Body: ioutil.NopCloser(bytes.NewBufferString(body)), - }, - nil -} - -func (c mockHttpClient) UseCertificate(certificates []tls.Certificate) {} - -func TestHttpRequest(t *testing.T) { - ActiveHttpClient = mockHttpClient{} - - req, err := http.NewRequest("GET", "http://host/okpath", nil) - assert.Nil(t, err) - response, err := HttpDo(req, time.Second*10, "description") - assert.Nil(t, err) - assert.Equal(t, 200, response.StatusCode) - - req, err = http.NewRequest("GET", "http://host/otherpath", nil) - assert.Nil(t, err) - response, err = HttpDo(req, time.Second*10, "description") - assert.Nil(t, err) - assert.Equal(t, 500, response.StatusCode) -} diff --git a/client/go/version/version.go b/client/go/version/version.go index b27529fa5e1..557007b18fc 100644 --- a/client/go/version/version.go +++ b/client/go/version/version.go @@ -68,6 +68,15 @@ func (v1 Version) Compare(v2 Version) int { // Less returns true if v1 is lower than v2. func (v1 Version) Less(v2 Version) bool { return v1.Compare(v2) < 0 } +// MustParse is like Parse, but panics if s cannot be parsed. +func MustParse(s string) Version { + v, err := Parse(s) + if err != nil { + panic(err) + } + return v +} + // Parse parses a semantic version number from string s. func Parse(s string) (Version, error) { if len(s) > 0 && s[0] == 'v' { diff --git a/client/go/vespa/crypto.go b/client/go/vespa/crypto.go index 25d3a937f4b..6f07df47b33 100644 --- a/client/go/vespa/crypto.go +++ b/client/go/vespa/crypto.go @@ -160,11 +160,15 @@ func ECPrivateKeyFrom(pemPrivateKey []byte) (*ecdsa.PrivateKey, error) { return nil, fmt.Errorf("invalid pem private key") } if privateKeyBlock.Type == "EC PRIVATE KEY" { - return x509.ParseECPrivateKey(privateKeyBlock.Bytes) // Raw EC private key + privateKey, err := x509.ParseECPrivateKey(privateKeyBlock.Bytes) // Raw EC private key + if err != nil { + return nil, fmt.Errorf("invalid raw ec private key: %w", err) + } + return privateKey, nil } privateKey, err := x509.ParsePKCS8PrivateKey(privateKeyBlock.Bytes) // Try PKCS8 format if err != nil { - return nil, err + return nil, fmt.Errorf("invalid pkcs8 private key: %w", err) } ecKey, ok := privateKey.(*ecdsa.PrivateKey) if !ok { diff --git a/client/go/vespa/deploy.go b/client/go/vespa/deploy.go index 2a6d51c7b9f..59ea8d22d62 100644 --- a/client/go/vespa/deploy.go +++ b/client/go/vespa/deploy.go @@ -43,6 +43,7 @@ type DeploymentOptions struct { Target Target ApplicationPackage ApplicationPackage Timeout time.Duration + HTTPClient util.HTTPClient } type LogLinePrepareResponse struct { @@ -124,7 +125,7 @@ func Prepare(deployment DeploymentOptions) (PrepareResult, error) { return PrepareResult{}, err } serviceDescription := "Deploy service" - response, err := util.HttpDo(req, time.Second*30, serviceDescription) + response, err := deployment.HTTPClient.Do(req, time.Second*30) if err != nil { return PrepareResult{}, err } @@ -149,7 +150,7 @@ func Activate(sessionID int64, deployment DeploymentOptions) error { return err } serviceDescription := "Deploy service" - response, err := util.HttpDo(req, time.Second*30, serviceDescription) + response, err := deployment.HTTPClient.Do(req, time.Second*30) if err != nil { return err } @@ -241,9 +242,9 @@ func Submit(opts DeploymentOptions) error { serviceDescription := "Submit service" sigKeyId := opts.Target.Deployment().Application.SerializedForm() if err := opts.Target.SignRequest(request, sigKeyId); err != nil { - return err + return fmt.Errorf("failed to sign api request: %w", err) } - response, err := util.HttpDo(request, time.Minute*10, sigKeyId) + response, err := opts.HTTPClient.Do(request, time.Minute*10) if err != nil { return err } diff --git a/client/go/vespa/target.go b/client/go/vespa/target.go index 8d48f1ffa3a..94028fb235f 100644 --- a/client/go/vespa/target.go +++ b/client/go/vespa/target.go @@ -45,6 +45,7 @@ type Service struct { Name string TLSOptions TLSOptions ztsClient ztsClient + httpClient util.HTTPClient } // Target represents a Vespa platform, running named Vespa services. @@ -89,7 +90,7 @@ type LogOptions struct { // Do sends request to this service. Any required authentication happens automatically. func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Response, error) { if s.TLSOptions.KeyPair.Certificate != nil { - util.ActiveHttpClient.UseCertificate([]tls.Certificate{s.TLSOptions.KeyPair}) + s.httpClient.UseCertificate([]tls.Certificate{s.TLSOptions.KeyPair}) } if s.TLSOptions.AthenzDomain != "" { accessToken, err := s.ztsClient.AccessToken(s.TLSOptions.AthenzDomain, s.TLSOptions.KeyPair) @@ -101,7 +102,7 @@ func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Respon } request.Header.Add("Authorization", "Bearer "+accessToken) } - return util.HttpDo(request, timeout, s.Description()) + return s.httpClient.Do(request, timeout) } // Wait polls the health check of this service until it succeeds or timeout passes. @@ -115,7 +116,7 @@ func (s *Service) Wait(timeout time.Duration) (int, error) { default: return 0, fmt.Errorf("invalid service: %s", s.Name) } - return waitForOK(url, &s.TLSOptions.KeyPair, timeout) + return waitForOK(s.httpClient, url, &s.TLSOptions.KeyPair, timeout) } func (s *Service) Description() string { @@ -138,18 +139,18 @@ type requestFunc func() *http.Request // waitForOK queries url and returns its status code. If the url returns a non-200 status code, it is repeatedly queried // until timeout elapses. -func waitForOK(url string, certificate *tls.Certificate, timeout time.Duration) (int, error) { +func waitForOK(client util.HTTPClient, url string, certificate *tls.Certificate, timeout time.Duration) (int, error) { req, err := http.NewRequest("GET", url, nil) if err != nil { return 0, err } okFunc := func(status int, response []byte) (bool, error) { return isOK(status), nil } - return wait(okFunc, func() *http.Request { return req }, certificate, timeout) + return wait(client, okFunc, func() *http.Request { return req }, certificate, timeout) } -func wait(fn responseFunc, reqFn requestFunc, certificate *tls.Certificate, timeout time.Duration) (int, error) { +func wait(client util.HTTPClient, fn responseFunc, reqFn requestFunc, certificate *tls.Certificate, timeout time.Duration) (int, error) { if certificate != nil { - util.ActiveHttpClient.UseCertificate([]tls.Certificate{*certificate}) + client.UseCertificate([]tls.Certificate{*certificate}) } var ( httpErr error @@ -160,7 +161,7 @@ func wait(fn responseFunc, reqFn requestFunc, certificate *tls.Certificate, time loopOnce := timeout == 0 for time.Now().Before(deadline) || loopOnce { req := reqFn() - response, httpErr = util.HttpDo(req, 10*time.Second, "") + response, httpErr = client.Do(req, 10*time.Second) if httpErr == nil { statusCode = response.StatusCode body, err := ioutil.ReadAll(response.Body) diff --git a/client/go/vespa/target_cloud.go b/client/go/vespa/target_cloud.go index f4eccaacab6..697b0f23ba1 100644 --- a/client/go/vespa/target_cloud.go +++ b/client/go/vespa/target_cloud.go @@ -11,10 +11,10 @@ import ( "strconv" "time" - "github.com/vespa-engine/vespa/client/go/auth0" + "github.com/vespa-engine/vespa/client/go/auth/auth0" + "github.com/vespa-engine/vespa/client/go/auth/zts" "github.com/vespa-engine/vespa/client/go/util" "github.com/vespa-engine/vespa/client/go/version" - "github.com/vespa-engine/vespa/client/go/zts" ) // CloudOptions configures URL and authentication for a cloud target. @@ -36,6 +36,7 @@ type cloudTarget struct { apiOptions APIOptions deploymentOptions CloudDeploymentOptions logOptions LogOptions + httpClient util.HTTPClient ztsClient ztsClient } @@ -67,12 +68,13 @@ type ztsClient interface { } // CloudTarget creates a Target for the Vespa Cloud or hosted Vespa platform. -func CloudTarget(apiOptions APIOptions, deploymentOptions CloudDeploymentOptions, logOptions LogOptions) (Target, error) { - ztsClient, err := zts.NewClient(zts.DefaultURL, util.ActiveHttpClient) +func CloudTarget(httpClient util.HTTPClient, apiOptions APIOptions, deploymentOptions CloudDeploymentOptions, logOptions LogOptions) (Target, error) { + ztsClient, err := zts.NewClient(zts.DefaultURL, httpClient) if err != nil { return nil, err } return &cloudTarget{ + httpClient: httpClient, apiOptions: apiOptions, deploymentOptions: deploymentOptions, logOptions: logOptions, @@ -117,7 +119,13 @@ func (t *cloudTarget) Deployment() Deployment { return t.deploymentOptions.Deplo func (t *cloudTarget) Service(name string, timeout time.Duration, runID int64, cluster string) (*Service, error) { switch name { case DeployService: - service := &Service{Name: name, BaseURL: t.apiOptions.System.URL, TLSOptions: t.apiOptions.TLSOptions, ztsClient: t.ztsClient} + service := &Service{ + Name: name, + BaseURL: t.apiOptions.System.URL, + TLSOptions: t.apiOptions.TLSOptions, + ztsClient: t.ztsClient, + httpClient: t.httpClient, + } if timeout > 0 { status, err := service.Wait(timeout) if err != nil { @@ -139,7 +147,13 @@ func (t *cloudTarget) Service(name string, timeout time.Duration, runID int64, c return nil, err } t.deploymentOptions.TLSOptions.AthenzDomain = t.apiOptions.System.AthenzDomain - return &Service{Name: name, BaseURL: url, TLSOptions: t.deploymentOptions.TLSOptions, ztsClient: t.ztsClient}, nil + return &Service{ + Name: name, + BaseURL: url, + TLSOptions: t.deploymentOptions.TLSOptions, + ztsClient: t.ztsClient, + httpClient: t.httpClient, + }, nil } return nil, fmt.Errorf("unknown service: %s", name) } @@ -168,7 +182,7 @@ func (t *cloudTarget) CheckVersion(clientVersion version.Version) error { if err != nil { return err } - response, err := util.HttpDo(req, 10*time.Second, "") + response, err := t.httpClient.Do(req, 10*time.Second) if err != nil { return err } @@ -254,7 +268,7 @@ func (t *cloudTarget) PrintLog(options LogOptions) error { if options.Follow { timeout = math.MaxInt64 // No timeout } - _, err = wait(logFunc, requestFunc, &t.apiOptions.TLSOptions.KeyPair, timeout) + _, err = wait(t.httpClient, logFunc, requestFunc, &t.apiOptions.TLSOptions.KeyPair, timeout) return err } @@ -305,7 +319,7 @@ func (t *cloudTarget) waitForRun(runID int64, timeout time.Duration) error { } return true, nil } - _, err = wait(jobSuccessFunc, requestFunc, &t.apiOptions.TLSOptions.KeyPair, timeout) + _, err = wait(t.httpClient, jobSuccessFunc, requestFunc, &t.apiOptions.TLSOptions.KeyPair, timeout) return err } @@ -363,7 +377,7 @@ func (t *cloudTarget) discoverEndpoints(timeout time.Duration) error { } return true, nil } - if _, err = wait(endpointFunc, func() *http.Request { return req }, &t.apiOptions.TLSOptions.KeyPair, timeout); err != nil { + if _, err = wait(t.httpClient, endpointFunc, func() *http.Request { return req }, &t.apiOptions.TLSOptions.KeyPair, timeout); err != nil { return err } if len(urlsByCluster) == 0 { diff --git a/client/go/vespa/target_custom.go b/client/go/vespa/target_custom.go index 072ec8649e4..bc25f19bf1a 100644 --- a/client/go/vespa/target_custom.go +++ b/client/go/vespa/target_custom.go @@ -7,12 +7,14 @@ import ( "net/url" "time" + "github.com/vespa-engine/vespa/client/go/util" "github.com/vespa-engine/vespa/client/go/version" ) type customTarget struct { targetType string baseURL string + httpClient util.HTTPClient } type serviceConvergeResponse struct { @@ -20,13 +22,13 @@ type serviceConvergeResponse struct { } // LocalTarget creates a target for a Vespa platform running locally. -func LocalTarget() Target { - return &customTarget{targetType: TargetLocal, baseURL: "http://127.0.0.1"} +func LocalTarget(httpClient util.HTTPClient) Target { + return &customTarget{targetType: TargetLocal, baseURL: "http://127.0.0.1", httpClient: httpClient} } // CustomTarget creates a Target for a Vespa platform running at baseURL. -func CustomTarget(baseURL string) Target { - return &customTarget{targetType: TargetCustom, baseURL: baseURL} +func CustomTarget(httpClient util.HTTPClient, baseURL string) Target { + return &customTarget{targetType: TargetCustom, baseURL: baseURL, httpClient: httpClient} } func (t *customTarget) Type() string { return t.targetType } @@ -40,7 +42,7 @@ func (t *customTarget) createService(name string) (*Service, error) { if err != nil { return nil, err } - return &Service{BaseURL: url, Name: name}, nil + return &Service{BaseURL: url, Name: name, httpClient: t.httpClient}, nil } return nil, fmt.Errorf("unknown service: %s", name) } @@ -118,7 +120,7 @@ func (t *customTarget) waitForConvergence(timeout time.Duration) error { converged = resp.Converged return converged, nil } - if _, err := wait(convergedFunc, func() *http.Request { return req }, nil, timeout); err != nil { + if _, err := wait(t.httpClient, convergedFunc, func() *http.Request { return req }, nil, timeout); err != nil { return err } if !converged { diff --git a/client/go/vespa/target_test.go b/client/go/vespa/target_test.go index bf3e0fae7d0..ca4ed86162c 100644 --- a/client/go/vespa/target_test.go +++ b/client/go/vespa/target_test.go @@ -13,6 +13,8 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/mock" + "github.com/vespa-engine/vespa/client/go/util" "github.com/vespa-engine/vespa/client/go/version" ) @@ -64,17 +66,17 @@ func (v *mockVespaApi) mockVespaHandler(w http.ResponseWriter, req *http.Request } func TestCustomTarget(t *testing.T) { - lt := LocalTarget() + lt := LocalTarget(&mock.HTTPClient{}) assertServiceURL(t, "http://127.0.0.1:19071", lt, "deploy") assertServiceURL(t, "http://127.0.0.1:8080", lt, "query") assertServiceURL(t, "http://127.0.0.1:8080", lt, "document") - ct := CustomTarget("http://192.0.2.42") + ct := CustomTarget(&mock.HTTPClient{}, "http://192.0.2.42") assertServiceURL(t, "http://192.0.2.42:19071", ct, "deploy") assertServiceURL(t, "http://192.0.2.42:8080", ct, "query") assertServiceURL(t, "http://192.0.2.42:8080", ct, "document") - ct2 := CustomTarget("http://192.0.2.42:60000") + ct2 := CustomTarget(&mock.HTTPClient{}, "http://192.0.2.42:60000") assertServiceURL(t, "http://192.0.2.42:60000", ct2, "deploy") assertServiceURL(t, "http://192.0.2.42:60000", ct2, "query") assertServiceURL(t, "http://192.0.2.42:60000", ct2, "document") @@ -84,7 +86,7 @@ func TestCustomTargetWait(t *testing.T) { vc := mockVespaApi{} srv := httptest.NewServer(http.HandlerFunc(vc.mockVespaHandler)) defer srv.Close() - target := CustomTarget(srv.URL) + target := CustomTarget(util.CreateClient(time.Second*10), srv.URL) _, err := target.Service("query", time.Millisecond, 42, "") assert.NotNil(t, err) @@ -147,17 +149,9 @@ func TestCheckVersion(t *testing.T) { defer srv.Close() target := createCloudTarget(t, srv.URL, ioutil.Discard) - assert.Nil(t, target.CheckVersion(mustVersion("8.0.0"))) - assert.Nil(t, target.CheckVersion(mustVersion("8.1.0"))) - assert.NotNil(t, target.CheckVersion(mustVersion("7.0.0"))) -} - -func mustVersion(s string) version.Version { - v, err := version.Parse(s) - if err != nil { - panic(err) - } - return v + assert.Nil(t, target.CheckVersion(version.MustParse("8.0.0"))) + assert.Nil(t, target.CheckVersion(version.MustParse("8.1.0"))) + assert.NotNil(t, target.CheckVersion(version.MustParse("7.0.0"))) } func createCloudTarget(t *testing.T, url string, logWriter io.Writer) Target { @@ -170,6 +164,7 @@ func createCloudTarget(t *testing.T, url string, logWriter io.Writer) Target { assert.Nil(t, err) target, err := CloudTarget( + util.CreateClient(time.Second*10), APIOptions{APIKey: apiKey, System: PublicSystem}, CloudDeploymentOptions{ Deployment: Deployment{ diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/http/ReindexingV1ApiHandler.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/http/ReindexingV1ApiHandler.java index 6b10cc911c2..8d9d21999d5 100644 --- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/http/ReindexingV1ApiHandler.java +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/http/ReindexingV1ApiHandler.java @@ -4,20 +4,17 @@ package ai.vespa.reindexing.http; import ai.vespa.reindexing.Reindexing; import ai.vespa.reindexing.ReindexingCurator; import com.google.inject.Inject; -import com.yahoo.cloud.config.ClusterListConfig; import com.yahoo.cloud.config.ZookeepersConfig; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.jdisc.Metric; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.Path; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.reindexing.ReindexingConfig; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.zookeeper.VespaZooKeeperServer; @@ -40,13 +37,14 @@ public class ReindexingV1ApiHandler extends ThreadedHttpRequestHandler { @Inject public ReindexingV1ApiHandler(Executor executor, Metric metric, - @SuppressWarnings("unused") VespaZooKeeperServer ensureZkHasStarted, ZookeepersConfig zookeepersConfig, - ReindexingConfig reindexingConfig, DocumentmanagerConfig documentmanagerConfig) { + @SuppressWarnings("unused") VespaZooKeeperServer ensureZkHasStarted, + ZookeepersConfig zookeepersConfig, + ReindexingConfig reindexingConfig, + DocumentTypeManager documentTypeManager) { this(executor, metric, reindexingConfig.clusters().keySet(), - new ReindexingCurator(Curator.create(zookeepersConfig.zookeeperserverlist()), - new DocumentTypeManager(documentmanagerConfig))); + new ReindexingCurator(Curator.create(zookeepersConfig.zookeeperserverlist()), documentTypeManager)); } ReindexingV1ApiHandler(Executor executor, Metric metric, Collection<String> clusterNames, ReindexingCurator database) { diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 7cb36374568..956ba2fdcf7 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -117,6 +117,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"bjorncs", "baldersheim"}) default boolean enableJdiscPreshutdownCommand() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default boolean avoidRenamingSummaryFeatures() { return false; } @ModelFeatureFlag(owners = {"bjorncs", "baldersheim"}) default boolean mergeGroupingResultInSearchInvoker() { return false; } + @ModelFeatureFlag(owners = {"arnej"}) default boolean experimentalSdParsing() { return false; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 51f3455762c..480b6590555 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -79,6 +79,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean useV8GeoPositions = false; private List<String> environmentVariables = List.of(); private boolean avoidRenamingSummaryFeatures = false; + private boolean experimentalSdParsing = false; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -138,6 +139,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } @Override public List<String> environmentVariables() { return environmentVariables; } @Override public boolean avoidRenamingSummaryFeatures() { return this.avoidRenamingSummaryFeatures; } + @Override public boolean experimentalSdParsing() { return this.experimentalSdParsing; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { this.maxUnCommittedMemory = maxUnCommittedMemory; @@ -375,6 +377,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setExperimentalSdParsing(boolean value) { + this.experimentalSdParsing = value; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java b/config-model/src/main/java/com/yahoo/searchdefinition/Application.java index 64688a7e70d..16eef798acd 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Application.java @@ -84,6 +84,7 @@ public class Application { List<Schema> schemasSomewhatOrdered = new ArrayList<>(schemas); for (Schema schema : new SearchOrderer().order(schemasSomewhatOrdered)) { + new FieldOperationApplierForStructs().processSchemaFields(schema); new FieldOperationApplierForSearch().process(schema); // TODO: Why is this not in the regular list? new Processing(properties).process(schema, logger, diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java index 533546b4d39..67d8d679275 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java @@ -15,6 +15,9 @@ import com.yahoo.io.reader.NamedReader; import com.yahoo.path.Path; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileXMLReader; +import com.yahoo.searchdefinition.parser.ConvertSchemaCollection; +import com.yahoo.searchdefinition.parser.IntermediateCollection; +import com.yahoo.searchdefinition.parser.IntermediateParser; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchdefinition.parser.SDParser; import com.yahoo.searchdefinition.parser.SimpleCharStream; @@ -43,6 +46,7 @@ import java.util.Set; */ public class ApplicationBuilder { + private final IntermediateCollection mediator; private final ApplicationPackage applicationPackage; private final List<Schema> schemas = new ArrayList<>(); private final DocumentTypeManager documentTypeManager = new DocumentTypeManager(); @@ -98,10 +102,17 @@ public class ApplicationBuilder { this(rankProfileRegistry, queryProfileRegistry, new TestProperties()); } + /** For testing only */ + public ApplicationBuilder(ModelContext.Properties properties) { + this(new RankProfileRegistry(), new QueryProfileRegistry(), properties); + } + + /** For testing only */ public ApplicationBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry, ModelContext.Properties properties) { this(MockApplicationPackage.createEmpty(), new MockFileRegistry(), new BaseDeployLogger(), properties, rankProfileRegistry, queryProfileRegistry); } + /** normal constructor */ public ApplicationBuilder(ApplicationPackage app, FileRegistry fileRegistry, DeployLogger deployLogger, @@ -118,6 +129,7 @@ public class ApplicationBuilder { RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry, boolean documentsOnly) { + this.mediator = new IntermediateCollection(deployLogger, properties); this.applicationPackage = applicationPackage; this.rankProfileRegistry = rankProfileRegistry; this.queryProfileRegistry = queryProfileRegistry; @@ -133,13 +145,17 @@ public class ApplicationBuilder { * Adds a schema to this application. * * @param fileName the name of the file to import - * @return the name of the imported object * @throws IOException thrown if the file can not be read for some reason * @throws ParseException thrown if the file does not contain a valid search definition */ - public Schema addSchemaFile(String fileName) throws IOException, ParseException { + public void addSchemaFile(String fileName) throws IOException, ParseException { + if (properties.featureFlags().experimentalSdParsing()) { + var parsedName = mediator.addSchemaFromFile(fileName); + addRankProfileFiles(parsedName); + return; + } File file = new File(fileName); - return addSchema(IOUtils.readFile(file)); + addSchema(IOUtils.readFile(file)); } /** @@ -149,8 +165,19 @@ public class ApplicationBuilder { * @param reader the reader whose content to import */ public void addSchema(NamedReader reader) { + if (properties.featureFlags().experimentalSdParsing()) { + try { + var parsedName = mediator.addSchemaFromReader(reader); + addRankProfileFiles(parsedName); + } catch (ParseException e) { + throw new IllegalArgumentException("Could not parse schema file '" + reader.getName() + "'", e); + } + return; + } try { - String schemaName = addSchema(IOUtils.readAll(reader)).getName(); + Schema schema = createSchema(IOUtils.readAll(reader)); + add(schema); + String schemaName = schema.getName(); String schemaFileName = stripSuffix(reader.getName(), ApplicationPackage.SD_NAME_SUFFIX); if ( ! schemaFileName.equals(schemaName)) { throw new IllegalArgumentException("The file containing schema '" + schemaName + "' must be named '" + @@ -176,8 +203,13 @@ public class ApplicationBuilder { * * @param schemaString the content of the schema */ - public Schema addSchema(String schemaString) throws ParseException { - return add(createSchema(schemaString)); + public void addSchema(String schemaString) throws ParseException { + if (properties.featureFlags().experimentalSdParsing()) { + var parsed = mediator.addSchemaFromString(schemaString); + addRankProfileFiles(parsed.name()); + return; + } + add(createSchema(schemaString)); } /** @@ -202,6 +234,9 @@ public class ApplicationBuilder { } private Schema parseSchema(String schemaString) throws ParseException { + if (properties.featureFlags().experimentalSdParsing()) { + throw new IllegalArgumentException("should use new parser only"); + } SimpleCharStream stream = new SimpleCharStream(schemaString); try { return parserOf(stream).schema(documentTypeManager); @@ -215,6 +250,10 @@ public class ApplicationBuilder { private void addRankProfileFiles(Schema schema) { if (applicationPackage == null) return; + if (properties.featureFlags().experimentalSdParsing()) { + throw new IllegalArgumentException("should use new parser only"); + } + Path legacyRankProfilePath = ApplicationPackage.SEARCH_DEFINITIONS_DIR.append(schema.getName()); for (NamedReader reader : applicationPackage.getFiles(legacyRankProfilePath, ".profile")) parseRankProfile(reader, schema); @@ -224,8 +263,28 @@ public class ApplicationBuilder { parseRankProfile(reader, schema); } + private void addRankProfileFiles(String schemaName) throws ParseException { + if (applicationPackage == null) return; + if (! properties.featureFlags().experimentalSdParsing()) { + throw new IllegalArgumentException("should use old parser only"); + } + + Path legacyRankProfilePath = ApplicationPackage.SEARCH_DEFINITIONS_DIR.append(schemaName); + for (NamedReader reader : applicationPackage.getFiles(legacyRankProfilePath, ".profile")) { + mediator.addRankProfileFile(schemaName, reader); + } + + Path rankProfilePath = ApplicationPackage.SCHEMAS_DIR.append(schemaName); + for (NamedReader reader : applicationPackage.getFiles(rankProfilePath, ".profile")) { + mediator.addRankProfileFile(schemaName, reader); + } + } + /** Parses the rank profile of the given reader and adds it to the rank profile registry for this schema. */ private void parseRankProfile(NamedReader reader, Schema schema) { + if (properties.featureFlags().experimentalSdParsing()) { + throw new IllegalArgumentException("should use new parser only"); + } try { SimpleCharStream stream = new SimpleCharStream(IOUtils.readAll(reader.getReader())); try { @@ -256,7 +315,19 @@ public class ApplicationBuilder { */ public Application build(boolean validate) { if (application != null) throw new IllegalStateException("Application already built"); - + if (properties.featureFlags().experimentalSdParsing()) { + var converter = new ConvertSchemaCollection(mediator, + documentTypeManager, + applicationPackage, + fileRegistry, + deployLogger, + properties, + rankProfileRegistry, + documentsOnly); + for (var schema : converter.convertToSchemas()) { + add(schema); + } + } application = new Application(applicationPackage, schemas, rankProfileRegistry, @@ -355,7 +426,7 @@ public class ApplicationBuilder { } /** - * Convenience factory methdd to create a SearchBuilder from multiple SD files. Only for testing. + * Convenience factory methods to create a SearchBuilder from multiple SD files. Only for testing. */ public static ApplicationBuilder createFromFiles(Collection<String> fileNames) throws IOException, ParseException { return createFromFiles(fileNames, new BaseDeployLogger()); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java index a29d66dc8f2..0db810d5933 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java @@ -34,6 +34,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; @@ -155,9 +156,9 @@ public class DocumentModelBuilder { private static void addSearchField(SDField field, SearchDef searchDef) { SearchField searchField = - new SearchField(field, - field.getIndices().containsKey(field.getName()) && field.getIndices().get(field.getName()).getType().equals(Index.Type.VESPA), - field.getAttributes().containsKey(field.getName())); + new SearchField(field, + field.getIndices().containsKey(field.getName()) && field.getIndices().get(field.getName()).getType().equals(Index.Type.VESPA), + field.getAttributes().containsKey(field.getName())); searchDef.add(searchField); // Add field to views @@ -252,9 +253,13 @@ public class DocumentModelBuilder { } else if (type instanceof StructDataType) { StructDataType dt = (StructDataType) type; for (com.yahoo.document.Field field : dt.getFields()) { - if (field.getDataType() != type) { - // XXX deprecated: - field.setDataType(resolveTemporariesRecurse(field.getDataType(), repo, docs, replacements)); + var ft = field.getDataType(); + if (ft != type) { + var newft = resolveTemporariesRecurse(ft, repo, docs, replacements); + if (ft != newft) { + // XXX deprecated: + field.setDataType(newft); + } } } } @@ -298,89 +303,6 @@ public class DocumentModelBuilder { return null; } - @SuppressWarnings("deprecation") - private static void specialHandleAnnotationReference(NewDocumentType docType, Field field) { - DataType fieldType = specialHandleAnnotationReferenceRecurse(docType, field.getName(), field.getDataType()); - if (fieldType == null) { - return; - } - field.setDataType(fieldType); // XXX deprecated - } - - private static DataType specialHandleAnnotationReferenceRecurse(NewDocumentType docType, String fieldName, - DataType dataType) { - if (dataType instanceof TemporaryAnnotationReferenceDataType) { - TemporaryAnnotationReferenceDataType refType = (TemporaryAnnotationReferenceDataType)dataType; - if (refType.getId() != 0) { - return null; - } - AnnotationType target = docType.getAnnotationType(refType.getTarget()); - if (target == null) { - throw new RetryLaterException("Annotation '" + refType.getTarget() + "' in reference '" + fieldName + - "' does not exist."); - } - dataType = new AnnotationReferenceDataType(target); - addType(docType, dataType); - return dataType; - } - else if (dataType instanceof MapDataType) { - MapDataType t = (MapDataType)dataType; - DataType valueType = specialHandleAnnotationReferenceRecurse(docType, fieldName, t.getValueType()); - if (valueType == null) { - return null; - } - var mapType = new MapDataType(t.getKeyType(), valueType, t.getId()); - addType(docType, mapType); - return mapType; - } - else if (dataType instanceof ArrayDataType) { - ArrayDataType t = (ArrayDataType) dataType; - DataType nestedType = specialHandleAnnotationReferenceRecurse(docType, fieldName, t.getNestedType()); - if (nestedType == null) { - return null; - } - var lstType = new ArrayDataType(nestedType, t.getId()); - addType(docType, lstType); - return lstType; - } - else if (dataType instanceof WeightedSetDataType) { - WeightedSetDataType t = (WeightedSetDataType) dataType; - DataType nestedType = specialHandleAnnotationReferenceRecurse(docType, fieldName, t.getNestedType()); - if (nestedType == null) { - return null; - } - boolean c = t.createIfNonExistent(); - boolean r = t.removeIfZero(); - var lstType = new WeightedSetDataType(nestedType, c, r, t.getId()); - addType(docType, lstType); - return lstType; - } - return null; - } - - private static StructDataType handleStruct(NewDocumentType dt, SDDocumentType type) { - StructDataType s = new StructDataType(type.getName()); - for (Field f : type.getDocumentType().contentStruct().getFieldsThisTypeOnly()) { - specialHandleAnnotationReference(dt, f); - s.addField(f); - } - for (StructDataType inherited : type.getDocumentType().contentStruct().getInheritedTypes()) { - s.inherit(inherited); - } - extractNestedTypes(dt, s); - addType(dt, s); - return s; - } - - private static StructDataType handleStruct(NewDocumentType dt, StructDataType s) { - for (Field f : s.getFieldsThisTypeOnly()) { - specialHandleAnnotationReference(dt, f); - } - extractNestedTypes(dt, s); - addType(dt, s); - return s; - } - private static boolean anyParentsHavePayLoad(SDAnnotationType sa, SDDocumentType sdoc) { if (sa.getInherits() != null) { AnnotationType tmp = sdoc.findAnnotation(sa.getInherits()); @@ -391,8 +313,6 @@ public class DocumentModelBuilder { } private NewDocumentType convert(SDDocumentType sdoc) { - Map<AnnotationType, String> annotationInheritance = new HashMap<>(); - Map<StructDataType, String> structInheritance = new HashMap<>(); NewDocumentType dt = new NewDocumentType(new NewDocumentType.Name(sdoc.getName()), sdoc.getDocumentType().contentStruct(), sdoc.getFieldSets(), @@ -400,63 +320,223 @@ public class DocumentModelBuilder { convertTemporaryImportedFieldsToNames(sdoc.getTemporaryImportedFields())); for (SDDocumentType n : sdoc.getInheritedTypes()) { NewDocumentType.Name name = new NewDocumentType.Name(n.getName()); - NewDocumentType inherited = model.getDocumentManager().getDocumentType(name); - if (inherited != null) { - dt.inherit(inherited); - } - } - for (SDDocumentType type : sdoc.getTypes()) { - if (type.isStruct()) { - handleStruct(dt, type); - } else { - throw new IllegalArgumentException("Data type '" + sdoc.getName() + "' is not a struct => tostring='" + sdoc.toString() + "'."); - } - } - for (SDDocumentType type : sdoc.getTypes()) { - for (SDDocumentType proxy : type.getInheritedTypes()) { - var inherited = dt.getDataTypeRecursive(proxy.getName()); - var converted = (StructDataType) dt.getDataType(type.getName()); - converted.inherit((StructDataType) inherited); + NewDocumentType inherited = model.getDocumentManager().getDocumentType(name); + if (inherited != null) { + dt.inherit(inherited); } } - for (AnnotationType annotation : sdoc.getAnnotations().values()) { - dt.add(annotation); + var extractor = new TypeExtractor(dt); + extractor.extract(sdoc); + return dt; + } + + static class TypeExtractor { + private final NewDocumentType targetDt; + Map<AnnotationType, String> annotationInheritance = new HashMap<>(); + Map<StructDataType, String> structInheritance = new HashMap<>(); + private final Map<Object, Object> inProgress = new IdentityHashMap<>(); + TypeExtractor(NewDocumentType target) { + this.targetDt = target; } - for (AnnotationType annotation : sdoc.getAnnotations().values()) { - SDAnnotationType sa = (SDAnnotationType) annotation; - if (annotation.getInheritedTypes().isEmpty() && (sa.getInherits() != null) ) { - annotationInheritance.put(annotation, sa.getInherits()); + + void extract(SDDocumentType sdoc) { + for (SDDocumentType type : sdoc.getTypes()) { + if (type.isStruct()) { + handleStruct(type); + } else { + throw new IllegalArgumentException("Data type '" + type.getName() + "' is not a struct => tostring='" + type.toString() + "'."); + } + } + for (SDDocumentType type : sdoc.getTypes()) { + for (SDDocumentType proxy : type.getInheritedTypes()) { + var inherited = targetDt.getDataTypeRecursive(proxy.getName()); + var converted = (StructDataType) targetDt.getDataType(type.getName()); + converted.inherit((StructDataType) inherited); + } } - if (annotation.getDataType() == null) { - if (sa.getSdDocType() != null) { - StructDataType s = handleStruct(dt, sa.getSdDocType()); - annotation.setDataType(s); - if ((sa.getInherits() != null)) { + for (AnnotationType annotation : sdoc.getAnnotations().values()) { + targetDt.add(annotation); + } + for (AnnotationType annotation : sdoc.getAnnotations().values()) { + SDAnnotationType sa = (SDAnnotationType) annotation; + if (annotation.getInheritedTypes().isEmpty() && (sa.getInherits() != null) ) { + annotationInheritance.put(annotation, sa.getInherits()); + } + if (annotation.getDataType() == null) { + if (sa.getSdDocType() != null) { + StructDataType s = handleStruct(sa.getSdDocType()); + annotation.setDataType(s); + if ((sa.getInherits() != null)) { + structInheritance.put(s, "annotation."+sa.getInherits()); + } + } else if (sa.getInherits() != null) { + StructDataType s = new StructDataType("annotation."+annotation.getName()); + if (anyParentsHavePayLoad(sa, sdoc)) { + annotation.setDataType(s); + addType(s); + } structInheritance.put(s, "annotation."+sa.getInherits()); } - } else if (sa.getInherits() != null) { - StructDataType s = new StructDataType("annotation."+annotation.getName()); - if (anyParentsHavePayLoad(sa, sdoc)) { - annotation.setDataType(s); - addType(dt, s); + } + } + for (Map.Entry<AnnotationType, String> e : annotationInheritance.entrySet()) { + e.getKey().inherit(targetDt.getAnnotationType(e.getValue())); + } + for (Map.Entry<StructDataType, String> e : structInheritance.entrySet()) { + StructDataType s = (StructDataType)targetDt.getDataType(e.getValue()); + if (s != null) { + e.getKey().inherit(s); + } + } + handleStruct(sdoc.getDocumentType().contentStruct()); + + extractDataTypesFromFields(sdoc.fieldSet()); + } + + private void extractDataTypesFromFields(Collection<Field> fields) { + for (Field f : fields) { + DataType type = f.getDataType(); + if (testAddType(type)) { + extractNestedTypes(type); + addType(type); + } + } + } + + private void extractNestedTypes(DataType type) { + if (inProgress.containsKey(type)) { + return; + } + inProgress.put(type, this); + if (type instanceof StructDataType) { + StructDataType tmp = (StructDataType) type; + extractDataTypesFromFields(tmp.getFieldsThisTypeOnly()); + } else if (type instanceof DocumentType) { + throw new IllegalArgumentException("Can not handle nested document definitions. In document type '" + targetDt.getName().toString() + + "', we can not define document type '" + type.toString()); + } else if (type instanceof CollectionDataType) { + CollectionDataType tmp = (CollectionDataType) type; + extractNestedTypes(tmp.getNestedType()); + addType(tmp.getNestedType()); + } else if (type instanceof MapDataType) { + MapDataType tmp = (MapDataType) type; + extractNestedTypes(tmp.getKeyType()); + extractNestedTypes(tmp.getValueType()); + addType(tmp.getKeyType()); + addType(tmp.getValueType()); + } else if (type instanceof TemporaryAnnotationReferenceDataType) { + throw new IllegalArgumentException(type.toString()); + } + } + + private boolean testAddType(DataType type) { return internalAddType(type, true); } + + private boolean addType(DataType type) { return internalAddType(type, false); } + + private boolean internalAddType(DataType type, boolean dryRun) { + DataType oldType = targetDt.getDataTypeRecursive(type.getId()); + if (oldType == null) { + if ( ! dryRun) { + targetDt.add(type); + } + return true; + } else if ((type instanceof StructDataType) && (oldType instanceof StructDataType)) { + StructDataType s = (StructDataType) type; + StructDataType os = (StructDataType) oldType; + if ((os.getFieldCount() == 0) && (s.getFieldCount() > os.getFieldCount())) { + if ( ! dryRun) { + targetDt.replace(type); } - structInheritance.put(s, "annotation."+sa.getInherits()); + return true; } } + return false; } - for (Map.Entry<AnnotationType, String> e : annotationInheritance.entrySet()) { - e.getKey().inherit(dt.getAnnotationType(e.getValue())); + + + @SuppressWarnings("deprecation") + private void specialHandleAnnotationReference(Field field) { + DataType fieldType = specialHandleAnnotationReferenceRecurse(field.getName(), field.getDataType()); + if (fieldType == null) { + return; + } + field.setDataType(fieldType); // XXX deprecated + } + + private DataType specialHandleAnnotationReferenceRecurse(String fieldName, + DataType dataType) { + if (dataType instanceof TemporaryAnnotationReferenceDataType) { + TemporaryAnnotationReferenceDataType refType = (TemporaryAnnotationReferenceDataType)dataType; + if (refType.getId() != 0) { + return null; + } + AnnotationType target = targetDt.getAnnotationType(refType.getTarget()); + if (target == null) { + throw new RetryLaterException("Annotation '" + refType.getTarget() + "' in reference '" + fieldName + + "' does not exist."); + } + dataType = new AnnotationReferenceDataType(target); + addType(dataType); + return dataType; + } + else if (dataType instanceof MapDataType) { + MapDataType t = (MapDataType)dataType; + DataType valueType = specialHandleAnnotationReferenceRecurse(fieldName, t.getValueType()); + if (valueType == null) { + return null; + } + var mapType = new MapDataType(t.getKeyType(), valueType, t.getId()); + addType(mapType); + return mapType; + } + else if (dataType instanceof ArrayDataType) { + ArrayDataType t = (ArrayDataType) dataType; + DataType nestedType = specialHandleAnnotationReferenceRecurse(fieldName, t.getNestedType()); + if (nestedType == null) { + return null; + } + var lstType = new ArrayDataType(nestedType, t.getId()); + addType(lstType); + return lstType; + } + else if (dataType instanceof WeightedSetDataType) { + WeightedSetDataType t = (WeightedSetDataType) dataType; + DataType nestedType = specialHandleAnnotationReferenceRecurse(fieldName, t.getNestedType()); + if (nestedType == null) { + return null; + } + boolean c = t.createIfNonExistent(); + boolean r = t.removeIfZero(); + var lstType = new WeightedSetDataType(nestedType, c, r, t.getId()); + addType(lstType); + return lstType; + } + return null; } - for (Map.Entry<StructDataType, String> e : structInheritance.entrySet()) { - StructDataType s = (StructDataType)dt.getDataType(e.getValue()); - if (s != null) { - e.getKey().inherit(s); + + private StructDataType handleStruct(SDDocumentType type) { + StructDataType s = new StructDataType(type.getName()); + for (Field f : type.getDocumentType().contentStruct().getFieldsThisTypeOnly()) { + specialHandleAnnotationReference(f); + s.addField(f); + } + for (StructDataType inherited : type.getDocumentType().contentStruct().getInheritedTypes()) { + s.inherit(inherited); } + extractNestedTypes(s); + addType(s); + return s; } - handleStruct(dt, sdoc.getDocumentType().contentStruct()); - extractDataTypesFromFields(dt, sdoc.fieldSet()); - return dt; + private StructDataType handleStruct(StructDataType s) { + for (Field f : s.getFieldsThisTypeOnly()) { + specialHandleAnnotationReference(f); + } + extractNestedTypes(s); + addType(s); + return s; + } + } private static Set<NewDocumentType.Name> convertDocumentReferencesToNames(Optional<DocumentReferences> documentReferences) { @@ -464,9 +544,9 @@ public class DocumentModelBuilder { return Set.of(); } return documentReferences.get().referenceMap().values().stream() - .map(documentReference -> documentReference.targetSearch().getDocument()) - .map(documentType -> new NewDocumentType.Name(documentType.getName())) - .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); + .map(documentReference -> documentReference.targetSearch().getDocument()) + .map(documentType -> new NewDocumentType.Name(documentType.getName())) + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); } private static Set<String> convertTemporaryImportedFieldsToNames(TemporaryImportedFields importedFields) { @@ -476,62 +556,6 @@ public class DocumentModelBuilder { return Collections.unmodifiableSet(importedFields.fields().keySet()); } - private static void extractDataTypesFromFields(NewDocumentType dt, Collection<Field> fields) { - for (Field f : fields) { - DataType type = f.getDataType(); - if (testAddType(dt, type)) { - extractNestedTypes(dt, type); - addType(dt, type); - } - } - } - - private static void extractNestedTypes(NewDocumentType dt, DataType type) { - if (type instanceof StructDataType) { - StructDataType tmp = (StructDataType) type; - extractDataTypesFromFields(dt, tmp.getFieldsThisTypeOnly()); - } else if (type instanceof DocumentType) { - throw new IllegalArgumentException("Can not handle nested document definitions. In document type '" + dt.getName().toString() + - "', we can not define document type '" + type.toString()); - } else if (type instanceof CollectionDataType) { - CollectionDataType tmp = (CollectionDataType) type; - extractNestedTypes(dt, tmp.getNestedType()); - addType(dt, tmp.getNestedType()); - } else if (type instanceof MapDataType) { - MapDataType tmp = (MapDataType) type; - extractNestedTypes(dt, tmp.getKeyType()); - extractNestedTypes(dt, tmp.getValueType()); - addType(dt, tmp.getKeyType()); - addType(dt, tmp.getValueType()); - } else if (type instanceof TemporaryAnnotationReferenceDataType) { - throw new IllegalArgumentException(type.toString()); - } - } - - private static boolean testAddType(NewDocumentType dt, DataType type) { return internalAddType(dt, type, true); } - - private static boolean addType(NewDocumentType dt, DataType type) { return internalAddType(dt, type, false); } - - private static boolean internalAddType(NewDocumentType dt, DataType type, boolean dryRun) { - DataType oldType = dt.getDataTypeRecursive(type.getId()); - if (oldType == null) { - if ( ! dryRun) { - dt.add(type); - } - return true; - } else if ((type instanceof StructDataType) && (oldType instanceof StructDataType)) { - StructDataType s = (StructDataType) type; - StructDataType os = (StructDataType) oldType; - if ((os.getFieldCount() == 0) && (s.getFieldCount() > os.getFieldCount())) { - if ( ! dryRun) { - dt.replace(type); - } - return true; - } - } - return false; - } - public static class RetryLaterException extends IllegalArgumentException { public RetryLaterException(String message) { super(message); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java index 16bf37902f5..5e5623e2319 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java @@ -35,17 +35,32 @@ public class FieldOperationApplierForStructs extends FieldOperationApplier { Iterator<Field> fields = anyType.fieldIterator(); while (fields.hasNext()) { SDField field = (SDField) fields.next(); - DataType structUsedByField = field.getFirstStructRecursive(); - if (structUsedByField == null) { - continue; - } - if (structUsedByField.getName().equals(structType.getName())) { - //this field is using this type!! - field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), 0); - field.populateWithStructMatching(sdoc, field.getDataType(), field.getMatching()); - } + maybePopulateField(sdoc, field, structType); } } } + private void maybePopulateField(SDDocumentType sdoc, SDField field, SDDocumentType structType) { + DataType structUsedByField = field.getFirstStructRecursive(); + if (structUsedByField == null) { + return; + } + if (structUsedByField.getName().equals(structType.getName())) { + //this field is using this type!! + field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), 0); + field.populateWithStructMatching(sdoc, field.getDataType(), field.getMatching()); + } + } + + public void processSchemaFields(Schema schema) { + var sdoc = schema.getDocument(); + if (sdoc == null) return; + for (SDDocumentType type : sdoc.getAllTypes()) { + if (type.isStruct()) { + for (SDField field : schema.allExtraFields()) { + maybePopulateField(sdoc, field, type); + } + } + } + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index e419b8c93a7..e7b7c92ca2f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -745,8 +745,8 @@ public class RankProfile implements Cloneable { public RankingExpressionFunction addFunction(ExpressionFunction function, boolean inline) { RankingExpressionFunction rankingExpressionFunction = new RankingExpressionFunction(function, inline); if (functions.containsKey(function.getName())) { - deployLogger.log(Level.WARNING, "Function '" + function.getName() + "' replaces a previous function " + - "with the same name in rank profile '" + this.name + "'"); + deployLogger.log(Level.WARNING, "Function '" + function.getName() + "' is defined twice " + + "in rank profile '" + this.name + "'"); } functions.put(function.getName(), rankingExpressionFunction); allFunctionsCached = null; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index d9bdf5dc917..0d1222e737b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -9,6 +9,7 @@ import com.yahoo.document.MapDataType; import com.yahoo.document.StructDataType; import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.document.TensorDataType; +import com.yahoo.document.WeightedSetDataType; import com.yahoo.language.Linguistics; import com.yahoo.language.process.Embedder; import com.yahoo.language.simple.SimpleLinguistics; @@ -186,6 +187,10 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, ": Dense tensor dimensions must have a size"); addQueryCommand("type " + type); } + else if (dataType instanceof WeightedSetDataType) { + var nested = ((WeightedSetDataType) dataType).getNestedType().getName(); + addQueryCommand("type WeightedSet<" + nested + ">"); + } else { addQueryCommand("type " + dataType.getName()); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java index dd697d51363..3cf5628d282 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java @@ -138,6 +138,7 @@ public class ConvertParsedFields { if (indexing.isPresent()) { field.setIndexingScript(indexing.get().script()); } + parsed.getWeight().ifPresent(value -> field.setWeight(value)); parsed.getStemming().ifPresent(value -> field.setStemming(value)); parsed.getNormalizing().ifPresent(value -> convertNormalizing(field, value)); for (var attribute : parsed.getAttributes()) { @@ -290,20 +291,23 @@ public class ConvertParsedFields { schema.addIndex(index); } - void convertStructDeclaration(Schema schema, SDDocumentType document, ParsedStruct parsed) { + SDDocumentType convertStructDeclaration(Schema schema, ParsedStruct parsed) { // TODO - can we cleanup this mess var structProxy = new SDDocumentType(parsed.name(), schema); structProxy.setStruct(context.resolveStruct(parsed)); - for (var structField : parsed.getFields()) { - var fieldType = context.resolveType(structField.getType()); - var field = new SDField(structProxy, structField.name(), fieldType); - convertCommonFieldSettings(field, structField); + for (var parsedField : parsed.getFields()) { + var fieldType = context.resolveType(parsedField.getType()); + var field = new SDField(structProxy, parsedField.name(), fieldType); + convertCommonFieldSettings(field, parsedField); structProxy.addField(field); + if (parsedField.hasIdOverride()) { + structProxy.setFieldId(field, parsedField.idOverride()); + } } for (String inherit : parsed.getInherited()) { structProxy.inherit(new DataTypeName(inherit)); } - document.addType(structProxy); + return structProxy; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedTypes.java index 5a83b4d8a0e..e67c1ac8275 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedTypes.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedTypes.java @@ -37,10 +37,12 @@ public class ConvertParsedTypes { this.docMan = docMan; } - public void convert() { + public void convert(boolean andRegister) { startDataTypes(); fillDataTypes(); - registerDataTypes(); + if (andRegister) { + registerDataTypes(); + } } private Map<String, DocumentType> documentsFromSchemas = new HashMap<>(); @@ -55,8 +57,8 @@ public class ConvertParsedTypes { for (var schema : orderedInput) { var doc = schema.getDocument(); for (var struct : doc.getStructs()) { - var dt = new StructDataType(struct.name()); String structId = doc.name() + "->" + struct.name(); + var dt = new StructDataType(struct.name()); structsFromSchemas.put(structId, dt); } for (var annotation : doc.getAnnotations()) { @@ -80,10 +82,20 @@ public class ConvertParsedTypes { for (var struct : doc.getStructs()) { String structId = doc.name() + "->" + struct.name(); var toFill = structsFromSchemas.get(structId); + // evil ugliness for (ParsedField field : struct.getFields()) { - var t = resolveFromContext(field.getType(), doc); - var f = new com.yahoo.document.Field(field.name(), t); - toFill.addField(f); + if (! field.hasIdOverride()) { + var t = resolveFromContext(field.getType(), doc); + var f = new com.yahoo.document.Field(field.name(), t); + toFill.addField(f); + } + } + for (ParsedField field : struct.getFields()) { + if (field.hasIdOverride()) { + var t = resolveFromContext(field.getType(), doc); + var f = new com.yahoo.document.Field(field.name(), field.idOverride(), t); + toFill.addField(f); + } } for (String inherit : struct.getInherited()) { var parent = findStructFromSchemas(inherit, doc); @@ -100,7 +112,9 @@ public class ConvertParsedTypes { var toFill = structsFromSchemas.get(structId); for (ParsedField field : struct.getFields()) { var t = resolveFromContext(field.getType(), doc); - var f = new com.yahoo.document.Field(field.name(), t); + var f = field.hasIdOverride() + ? new com.yahoo.document.Field(field.name(), field.idOverride(), t) + : new com.yahoo.document.Field(field.name(), t); toFill.addField(f); } at.setDataType(toFill); @@ -116,8 +130,11 @@ public class ConvertParsedTypes { for (var docField : doc.getFields()) { String name = docField.name(); var t = resolveFromContext(docField.getType(), doc); - var f = new com.yahoo.document.Field(name, t); + var f = new com.yahoo.document.Field(docField.name(), t); docToFill.addField(f); + if (docField.hasIdOverride()) { + f.setId(docField.idOverride(), docToFill); + } inDocFields.add(name); } fieldSets.put("[document]", inDocFields); @@ -171,18 +188,6 @@ public class ConvertParsedTypes { throw new IllegalArgumentException("conflicting values for struct " + name + " in " +doc); } } - if (found == null) { - // TODO: be more restrictive here, but we need something - // for imported fields. For now, fall back to looking for - // struct in any schema. - for (var schema : orderedInput) { - for (var struct : schema.getDocument().getStructs()) { - if (struct.name().equals(name)) { - return struct; - } - } - } - } return found; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java index fb0003ca4f9..21a68744c19 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java @@ -1,25 +1,43 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.parser; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; +import com.yahoo.document.DataTypeName; import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.PositionDataType; import com.yahoo.document.ReferenceDataType; import com.yahoo.document.StructDataType; -import com.yahoo.document.PositionDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; +import com.yahoo.searchdefinition.DefaultRankProfile; +import com.yahoo.searchdefinition.DocumentOnlySchema; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Schema; +import com.yahoo.searchdefinition.UnrankedRankProfile; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.TemporaryImportedField; +import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; +import com.yahoo.searchdefinition.parser.ConvertParsedTypes.TypeResolver; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; import java.util.List; -import java.util.Map; +import java.util.Optional; /** * Class converting a collection of schemas from the intermediate format. - * For now only conversion to DocumentType (with contents). * * @author arnej27959 **/ @@ -28,12 +46,45 @@ public class ConvertSchemaCollection { private final IntermediateCollection input; private final List<ParsedSchema> orderedInput = new ArrayList<>(); private final DocumentTypeManager docMan; + private final ApplicationPackage applicationPackage; + private final FileRegistry fileRegistry; + private final DeployLogger deployLogger; + private final ModelContext.Properties properties; + private final RankProfileRegistry rankProfileRegistry; + private final boolean documentsOnly; + + // for unit test + ConvertSchemaCollection(IntermediateCollection input, + DocumentTypeManager documentTypeManager) + { + this(input, documentTypeManager, + MockApplicationPackage.createEmpty(), + new MockFileRegistry(), + new BaseDeployLogger(), + new TestProperties(), + new RankProfileRegistry(), + true); + } public ConvertSchemaCollection(IntermediateCollection input, - DocumentTypeManager documentTypeManager) + DocumentTypeManager documentTypeManager, + ApplicationPackage applicationPackage, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + boolean documentsOnly) { this.input = input; this.docMan = documentTypeManager; + this.applicationPackage = applicationPackage; + this.fileRegistry = fileRegistry; + this.deployLogger = deployLogger; + this.properties = properties; + this.rankProfileRegistry = rankProfileRegistry; + this.documentsOnly = documentsOnly; + + input.resolveInternalConnections(); order(); pushTypesToDocuments(); } @@ -64,8 +115,161 @@ public class ConvertSchemaCollection { } } + private ConvertParsedTypes typeConverter; + public void convertTypes() { - var converter = new ConvertParsedTypes(orderedInput, docMan); - converter.convert(); + typeConverter = new ConvertParsedTypes(orderedInput, docMan); + typeConverter.convert(true); } + + public List<Schema> convertToSchemas() { + typeConverter = new ConvertParsedTypes(orderedInput, docMan); + typeConverter.convert(false); + var resultList = new ArrayList<Schema>(); + for (var parsed : orderedInput) { + Optional<String> inherited; + var inheritList = parsed.getInherited(); + if (inheritList.size() == 0) { + inherited = Optional.empty(); + } else if (inheritList.size() == 1) { + inherited = Optional.of(inheritList.get(0)); + } else { + throw new IllegalArgumentException("schema " + parsed.name() + "cannot inherit more than once"); + } + Schema schema = parsed.getDocumentWithoutSchema() + ? new DocumentOnlySchema(applicationPackage, fileRegistry, deployLogger, properties) + : new Schema(parsed.name(), applicationPackage, inherited, fileRegistry, deployLogger, properties); + convertSchema(schema, parsed); + resultList.add(schema); + } + return resultList; + } + + private void convertAnnotation(Schema schema, SDDocumentType document, ParsedAnnotation parsed, ConvertParsedFields fieldConverter) { + var type = new SDAnnotationType(parsed.name()); + for (String inherit : parsed.getInherited()) { + type.inherit(inherit); + } + var payload = parsed.getStruct(); + if (payload.isPresent()) { + var struct = fieldConverter.convertStructDeclaration(schema, payload.get()); + type = new SDAnnotationType(parsed.name(), struct, type.getInherits()); + // WTF? + struct.setStruct(null); + } + document.addAnnotation(type); + } + + private void convertDocument(Schema schema, ParsedDocument parsed, + ConvertParsedFields fieldConverter) + { + SDDocumentType document = new SDDocumentType(parsed.name()); + for (String inherit : parsed.getInherited()) { + document.inherit(new DataTypeName(inherit)); + } + for (var struct : parsed.getStructs()) { + var structProxy = fieldConverter.convertStructDeclaration(schema, struct); + document.addType(structProxy); + } + for (var annotation : parsed.getAnnotations()) { + convertAnnotation(schema, document, annotation, fieldConverter); + } + for (var field : parsed.getFields()) { + var sdf = fieldConverter.convertDocumentField(schema, document, field); + if (field.hasIdOverride()) { + document.setFieldId(sdf, field.idOverride()); + } + } + schema.addDocument(document); + } + + private void convertDocumentSummary(Schema schema, ParsedDocumentSummary parsed, TypeResolver typeContext) { + var docsum = new DocumentSummary(parsed.name(), schema); + var inheritList = parsed.getInherited(); + if (inheritList.size() == 1) { + docsum.setInherited(inheritList.get(0)); + } else if (inheritList.size() != 0) { + throw new IllegalArgumentException("document-summary "+parsed.name()+" cannot inherit more than once"); + } + if (parsed.getFromDisk()) { + docsum.setFromDisk(true); + } + if (parsed.getOmitSummaryFeatures()) { + docsum.setOmitSummaryFeatures(true); + } + for (var parsedField : parsed.getSummaryFields()) { + DataType dataType = typeContext.resolveType(parsedField.getType()); + var summaryField = new SummaryField(parsedField.name(), dataType); + // XXX does not belong here: + summaryField.setVsmCommand(SummaryField.VsmCommand.FLATTENSPACE); + ConvertParsedFields.convertSummaryFieldSettings(summaryField, parsedField); + docsum.add(summaryField); + } + schema.addSummary(docsum); + } + + private void convertImportField(Schema schema, ParsedSchema.ImportedField f) { + // needs rethinking + var importedFields = schema.temporaryImportedFields().get(); + if (importedFields.hasField(f.asFieldName)) { + throw new IllegalArgumentException("For schema '" + schema.getName() + + "', import field as '" + f.asFieldName + + "': Field already imported"); + } + importedFields.add(new TemporaryImportedField(f.asFieldName, f.refFieldName, f.foreignFieldName)); + } + + private void convertFieldSet(Schema schema, ParsedFieldSet parsed) { + String setName = parsed.name(); + for (String field : parsed.getFieldNames()) { + schema.fieldSets().addUserFieldSetItem(setName, field); + } + for (String command : parsed.getQueryCommands()) { + schema.fieldSets().userFieldSets().get(setName).queryCommands().add(command); + } + if (parsed.getMatchSettings().isPresent()) { + // same ugliness as SDParser.jj used to have: + var tmp = new SDField(setName, DataType.STRING); + ConvertParsedFields.convertMatchSettings(tmp, parsed.matchSettings()); + schema.fieldSets().userFieldSets().get(setName).setMatching(tmp.getMatching()); + } + } + + private void convertSchema(Schema schema, ParsedSchema parsed) { + if (parsed.hasStemming()) { + schema.setStemming(parsed.getStemming()); + } + parsed.getRawAsBase64().ifPresent(value -> schema.enableRawAsBase64(value)); + var typeContext = typeConverter.makeContext(parsed.getDocument()); + var fieldConverter = new ConvertParsedFields(typeContext); + convertDocument(schema, parsed.getDocument(), fieldConverter); + for (var field : parsed.getFields()) { + fieldConverter.convertExtraField(schema, field); + } + for (var index : parsed.getIndexes()) { + fieldConverter.convertExtraIndex(schema, index); + } + for (var docsum : parsed.getDocumentSummaries()) { + convertDocumentSummary(schema, docsum, typeContext); + } + for (var importedField : parsed.getImportedFields()) { + convertImportField(schema, importedField); + } + for (var fieldSet : parsed.getFieldSets()) { + convertFieldSet(schema, fieldSet); + } + for (var rankingConstant : parsed.getRankingConstants()) { + schema.rankingConstants().add(rankingConstant); + } + for (var onnxModel : parsed.getOnnxModels()) { + schema.onnxModels().add(onnxModel); + } + rankProfileRegistry.add(new DefaultRankProfile(schema, rankProfileRegistry, schema.rankingConstants())); + rankProfileRegistry.add(new UnrankedRankProfile(schema, rankProfileRegistry, schema.rankingConstants())); + var rankConverter = new ConvertParsedRanking(rankProfileRegistry); + for (var rankProfile : parsed.getRankProfiles()) { + rankConverter.convertRankProfile(schema, rankProfile); + } + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/InheritanceResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/InheritanceResolver.java index d9132d3aa24..4d011c1b596 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/InheritanceResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/InheritanceResolver.java @@ -15,6 +15,7 @@ public class InheritanceResolver { private final Map<String, ParsedSchema> parsedSchemas; private final Map<String, ParsedDocument> parsedDocs = new HashMap<>(); + private final Map<String, ParsedSchema> schemaForDocs = new HashMap<>(); public InheritanceResolver(Map<String, ParsedSchema> parsedSchemas) { this.parsedSchemas = parsedSchemas; @@ -24,7 +25,7 @@ public class InheritanceResolver { String name = schema.name(); if (seen.contains(name)) { seen.add(name); - throw new IllegalArgumentException("Inheritance cycle for schemas: " + + throw new IllegalArgumentException("Inheritance/reference cycle for schemas: " + String.join(" -> ", seen)); } seen.add(name); @@ -64,9 +65,13 @@ public class InheritanceResolver { if (old != null) { throw new IllegalArgumentException("duplicate document declaration for " + doc.name()); } + schemaForDocs.put(doc.name(), schema); for (String docInherit : doc.getInherited()) { schema.inheritByDocument(docInherit); } + for (String docReferenced : doc.getReferencedDocuments()) { + schema.inheritByDocument(docReferenced); + } } for (ParsedDocument doc : parsedDocs.values()) { for (String inherit : doc.getInherited()) { @@ -76,13 +81,20 @@ public class InheritanceResolver { } doc.resolveInherit(inherit, parentDoc); } + for (String docRefName : doc.getReferencedDocuments()) { + var refDoc = parsedDocs.get(docRefName); + if (refDoc == null) { + throw new IllegalArgumentException("document " + doc.name() + " references unavailable document " + docRefName); + } + doc.resolveReferenced(refDoc); + } } for (ParsedSchema schema : parsedSchemas.values()) { - for (String inherit : schema.getInheritedByDocument()) { - var parent = parsedSchemas.get(inherit); + for (String docName : schema.getInheritedByDocument()) { + var parent = schemaForDocs.get(docName); assert(parent.hasDocument()); - assert(parent.getDocument().name().equals(inherit)); - schema.resolveInheritByDocument(inherit, parent); + assert(parent.getDocument().name().equals(docName)); + schema.resolveInheritByDocument(docName, parent); } } } @@ -91,7 +103,7 @@ public class InheritanceResolver { String name = document.name(); if (seen.contains(name)) { seen.add(name); - throw new IllegalArgumentException("Inheritance cycle for documents: " + + throw new IllegalArgumentException("Inheritance/reference cycle for documents: " + String.join(" -> ", seen)); } seen.add(name); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java index 065b66e22b1..ed975238067 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.parser; - import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashMap; @@ -34,6 +33,19 @@ public class ParsedDocument extends ParsedBlock { ParsedStruct getStruct(String name) { return docStructs.get(name); } ParsedAnnotation getAnnotation(String name) { return docAnnotations.get(name); } + List<String> getReferencedDocuments() { + var result = new ArrayList<String>(); + for (var field : docFields.values()) { + var type = field.getType(); + if (type.getVariant() == ParsedType.Variant.DOC_REFERENCE) { + var docType = type.getReferencedDocumentType(); + assert(docType.getVariant() == ParsedType.Variant.DOCUMENT); + result.add(docType.name()); + } + } + return result; + } + void inherit(String other) { inherited.add(other); } void addField(ParsedField field) { @@ -54,6 +66,7 @@ public class ParsedDocument extends ParsedBlock { verifyThat(! docAnnotations.containsKey(annName), "already has annotation", annName); docAnnotations.put(annName, annotation); annotation.tagOwner(name()); + annotation.getStruct().ifPresent(s -> s.tagOwner(name())); } public String toString() { return "document " + name(); } @@ -64,5 +77,12 @@ public class ParsedDocument extends ParsedBlock { verifyThat(! resolvedInherits.containsKey(name), "double resolveInherit for", name); resolvedInherits.put(name, parsed); } + + void resolveReferenced(ParsedDocument parsed) { + // TODO - not really inheritance: + var old = resolvedInherits.put(parsed.name(), parsed); + assert(old == null || old == parsed); + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocumentSummary.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocumentSummary.java index 08f4946a218..25adc6f134f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocumentSummary.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocumentSummary.java @@ -28,10 +28,11 @@ class ParsedDocumentSummary extends ParsedBlock { List<ParsedSummaryField> getSummaryFields() { return List.copyOf(fields.values()); } List<String> getInherited() { return List.copyOf(inherited); } - void addField(ParsedSummaryField field) { + ParsedSummaryField addField(ParsedSummaryField field) { String fieldName = field.name(); - verifyThat(! fields.containsKey(fieldName), "already has field", fieldName); - fields.put(fieldName, field); + // TODO disallow this on Vespa 8 + // verifyThat(! fields.containsKey(fieldName), "already has field", fieldName); + return fields.put(fieldName, field); } void setFromDisk(boolean value) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java index 5ee73abc28d..ca876997dc6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java @@ -58,6 +58,7 @@ class ParsedField extends ParsedBlock { List<String> getQueryCommands() { return List.copyOf(queryCommands); } String lookupAliasedFrom(String alias) { return aliases.get(alias); } ParsedMatchSettings matchSettings() { return this.matchInfo; } + Optional<Integer> getWeight() { return Optional.ofNullable(weight); } Optional<Stemming> getStemming() { return Optional.ofNullable(stemming); } Optional<String> getNormalizing() { return Optional.ofNullable(normalizing); } Optional<ParsedIndexingOp> getIndexing() { return Optional.ofNullable(indexingOp); } @@ -121,10 +122,8 @@ class ParsedField extends ParsedBlock { void setStemming(Stemming stemming) { this.stemming = stemming; } void setWeight(int weight) { this.weight = weight; } - void addAttribute(ParsedAttribute attribute) { - String attrName = attribute.name(); - verifyThat(! attributes.containsKey(attrName), "already has attribute", attrName); - attributes.put(attrName, attribute); + ParsedAttribute attributeFor(String attrName) { + return attributes.computeIfAbsent(attrName, n -> new ParsedAttribute(n)); } void setIndexingOperation(ParsedIndexingOp idxOp) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java index 0801b613530..f028685b71a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java @@ -113,9 +113,10 @@ class ParsedRankProfile extends ParsedBlock { fieldsRankWeight.put(field, weight); } - void addFunction(ParsedRankFunction func) { - verifyThat(! functions.containsKey(func.name()), "already has function", func.name()); - functions.put(func.name(), func); + ParsedRankFunction addOrReplaceFunction(ParsedRankFunction func) { + // allowed with warning + // verifyThat(! functions.containsKey(func.name()), "already has function", func.name()); + return functions.put(func.name(), func); } void addMutateOperation(MutateOperation.Phase phase, String attrName, String operation) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java index bcbf14d9398..599dd6e2a7a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java @@ -9,6 +9,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; /** * This class holds the extracted information after parsing @@ -30,7 +31,7 @@ public class ParsedSchema extends ParsedBlock { } private boolean documentWithoutSchema = false; - private boolean rawAsBase64 = false; // TODO Vespa 8 flip default + private Boolean rawAsBase64 = null; private ParsedDocument myDocument = null; private Stemming defaultStemming = null; private final List<ImportedField> importedFields = new ArrayList<>(); @@ -53,7 +54,7 @@ public class ParsedSchema extends ParsedBlock { } boolean getDocumentWithoutSchema() { return documentWithoutSchema; } - boolean getRawAsBase64() { return rawAsBase64; } + Optional<Boolean> getRawAsBase64() { return Optional.ofNullable(rawAsBase64); } boolean hasDocument() { return myDocument != null; } ParsedDocument getDocument() { return myDocument; } boolean hasStemming() { return defaultStemming != null; } @@ -82,8 +83,9 @@ public class ParsedSchema extends ParsedBlock { void addDocument(ParsedDocument document) { verifyThat(myDocument == null, "already has", myDocument, "so cannot add", document); - verifyThat(name().equals(document.name()), - "schema " + name() + "can only contain document named " + name() + ", was: "+ document.name()); + // TODO - disallow? + // verifyThat(name().equals(document.name()), + // "schema " + name() + " can only contain document named " + name() + ", was: "+ document.name()); this.myDocument = document; } @@ -156,16 +158,15 @@ public class ParsedSchema extends ParsedBlock { verifyThat(name.equals(parsed.name()), "resolveInherit name mismatch for", name); verifyThat(! resolvedInherits.containsKey(name), "double resolveInherit for", name); resolvedInherits.put(name, parsed); - var old = allResolvedInherits.put(name, parsed); + var old = allResolvedInherits.put("schema " + name, parsed); verifyThat(old == null || old == parsed, "conflicting resolveInherit for", name); } void resolveInheritByDocument(String name, ParsedSchema parsed) { verifyThat(inheritedByDocument.contains(name), "resolveInheritByDocument for non-inherited name", name); - verifyThat(name.equals(parsed.name()), "resolveInheritByDocument name mismatch for", name); - var old = allResolvedInherits.put(name, parsed); - verifyThat(old == null || old == parsed, "conflicting resolveInherit for", name); + var old = allResolvedInherits.put("document " + name, parsed); + verifyThat(old == null || old == parsed, "conflicting resolveInheritByDocument for", name); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedStruct.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedStruct.java index 17b20459c9c..b5f297cf5da 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedStruct.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedStruct.java @@ -15,10 +15,13 @@ import java.util.Map; public class ParsedStruct extends ParsedBlock { private final List<String> inherited = new ArrayList<>(); private final Map<String, ParsedField> fields = new LinkedHashMap<>(); + private final ParsedType asParsedType; private String ownedBy = null; public ParsedStruct(String name) { super(name, "struct"); + this.asParsedType = ParsedType.fromName(name); + asParsedType.setVariant(ParsedType.Variant.STRUCT); } List<ParsedField> getFields() { return List.copyOf(fields.values()); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java index 3aed90a58e1..d04277706a1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java @@ -46,6 +46,7 @@ class ParsedType { case "raw": return Variant.BUILTIN; case "tag": return Variant.BUILTIN; case "position": return Variant.POSITION; + case "float16": return Variant.BUILTIN; } return Variant.UNKNOWN; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 20c3e007e3b..dbc055ef02e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -28,6 +28,8 @@ import com.yahoo.vespa.model.container.PlatformBundles; import java.util.Set; import java.util.TreeSet; +import static com.yahoo.vespa.model.container.docproc.DocprocChains.DOCUMENT_TYPE_MANAGER_CLASS; + /** * Container implementation for cluster-controllers */ @@ -148,6 +150,8 @@ public class ClusterControllerContainer extends Container implements addComponent("reindexing-maintainer", "ai.vespa.reindexing.ReindexingMaintainer", REINDEXING_CONTROLLER_BUNDLE); + + addComponent(new SimpleComponent(DOCUMENT_TYPE_MANAGER_CLASS)); addHandler("reindexing-status", "ai.vespa.reindexing.http.ReindexingV1ApiHandler", "/reindexing/v1/*", diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 8f95e390b07..fae12a63427 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -36,6 +36,7 @@ import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.configserver.ConfigserverCluster; +import com.yahoo.vespa.model.container.docproc.DocprocChains; import com.yahoo.vespa.model.utils.FileSender; import java.util.ArrayList; @@ -48,6 +49,7 @@ import java.util.Set; import java.util.stream.Collectors; import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.sharedLayer4; +import static com.yahoo.vespa.model.container.docproc.DocprocChains.DOCUMENT_TYPE_MANAGER_CLASS; /** * A container cluster that is typically set up from the user application. @@ -110,6 +112,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat addSimpleComponent("com.yahoo.container.jdisc.CertificateStoreProvider"); addSimpleComponent("com.yahoo.container.jdisc.AthenzIdentityProviderProvider"); addSimpleComponent(com.yahoo.container.core.documentapi.DocumentAccessProvider.class.getName()); + addSimpleComponent(DOCUMENT_TYPE_MANAGER_CLASS); addMetricsHandlers(); addTestrunnerComponentsIfTester(deployState); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java index 09995f661e4..4b9897d0950 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java @@ -7,6 +7,7 @@ import com.yahoo.container.jdisc.config.SessionConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.chain.Chains; import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; @@ -15,19 +16,28 @@ import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; * @author Einar M R Rosenvinge */ public class DocprocChains extends Chains<DocprocChain> { + + public static final String DOCUMENT_TYPE_MANAGER_CLASS = "com.yahoo.document.DocumentTypeManager"; + private final ProcessingHandler<DocprocChains> docprocHandler; - public DocprocChains(AbstractConfigProducer parent, String subId) { + public DocprocChains(AbstractConfigProducer<?> parent, String subId) { super(parent, subId); docprocHandler = new ProcessingHandler<>(this, "com.yahoo.docproc.jdisc.DocumentProcessingHandler"); addComponent(docprocHandler); + + if (! (getParent() instanceof ApplicationContainerCluster)) { + // All application containers already have a DocumentTypeManager, + // but this could also belong to e.g. a cluster controller. + addComponent(new SimpleComponent(DOCUMENT_TYPE_MANAGER_CLASS)); + } } - private void addComponent(Component component) { - if (!(getParent() instanceof ContainerCluster)) { + private void addComponent(Component<?, ?> component) { + if (!(getParent() instanceof ContainerCluster<?>)) { return; } - ((ContainerCluster) getParent()).addComponent(component); + ((ContainerCluster<?>) getParent()).addComponent(component); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java index 2742dc59fcd..88139de7888 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java @@ -36,15 +36,13 @@ import java.util.stream.Collectors; */ public class OnnxModelInfo { - private final ApplicationPackage app; private final String modelPath; private final String defaultOutput; private final Map<String, OnnxTypeInfo> inputs; private final Map<String, OnnxTypeInfo> outputs; private final Map<String, TensorType> vespaTypes = new HashMap<>(); - private OnnxModelInfo(ApplicationPackage app, String path, Map<String, OnnxTypeInfo> inputs, Map<String, OnnxTypeInfo> outputs, String defaultOutput) { - this.app = app; + private OnnxModelInfo(String path, Map<String, OnnxTypeInfo> inputs, Map<String, OnnxTypeInfo> outputs, String defaultOutput) { this.modelPath = path; this.inputs = Collections.unmodifiableMap(inputs); this.outputs = Collections.unmodifiableMap(outputs); @@ -81,15 +79,7 @@ public class OnnxModelInfo { Set<Long> unboundSizes = new HashSet<>(); Map<String, Long> symbolicSizes = new HashMap<>(); resolveUnknownDimensionSizes(inputTypes, symbolicSizes, unboundSizes); - - TensorType type = TensorType.empty; - if (inputTypes.size() > 0 && onnxTypeInfo.needModelProbe(symbolicSizes)) { - type = OnnxModelProbe.probeModel(app, Path.fromString(modelPath), onnxName, inputTypes); - } - if (type.equals(TensorType.empty)) { - type = onnxTypeInfo.toVespaTensorType(symbolicSizes, unboundSizes); - } - return type; + return onnxTypeInfo.toVespaTensorType(symbolicSizes, unboundSizes); } return vespaTypes.computeIfAbsent(onnxName, v -> onnxTypeInfo.toVespaTensorType()); } @@ -160,8 +150,7 @@ public class OnnxModelInfo { Onnx.ModelProto model = Onnx.ModelProto.parseFrom(inputStream); String json = onnxModelToJson(model, path); storeGeneratedInfo(json, path, app); - return jsonToModelInfo(json, app); - + return jsonToModelInfo(json); } catch (IOException e) { throw new IllegalArgumentException("Unable to parse ONNX model", e); } @@ -170,7 +159,7 @@ public class OnnxModelInfo { static private OnnxModelInfo loadFromGeneratedInfo(Path path, ApplicationPackage app) { try { String json = readGeneratedInfo(path, app); - return jsonToModelInfo(json, app); + return jsonToModelInfo(json); } catch (IOException e) { throw new IllegalArgumentException("Unable to parse ONNX model", e); } @@ -213,7 +202,7 @@ public class OnnxModelInfo { return out.toString(); } - static public OnnxModelInfo jsonToModelInfo(String json, ApplicationPackage app) throws IOException { + static public OnnxModelInfo jsonToModelInfo(String json) throws IOException { ObjectMapper m = new ObjectMapper(); JsonNode root = m.readTree(json); Map<String, OnnxTypeInfo> inputs = new HashMap<>(); @@ -233,7 +222,7 @@ public class OnnxModelInfo { if (root.get("outputs").has(0)) { defaultOutput = root.get("outputs").get(0).get("name").textValue(); } - return new OnnxModelInfo(app, path, inputs, outputs, defaultOutput); + return new OnnxModelInfo(path, inputs, outputs, defaultOutput); } static private void onnxTypeToJson(JsonGenerator g, Onnx.ValueInfoProto valueInfo) throws IOException { @@ -364,21 +353,6 @@ public class OnnxModelInfo { return builder.build(); } - boolean needModelProbe(Map<String, Long> symbolicSizes) { - for (OnnxDimensionInfo onnxDimension : dimensions) { - if (onnxDimension.hasSymbolicName()) { - if (symbolicSizes == null) - return true; - if ( ! symbolicSizes.containsKey(onnxDimension.getSymbolicName())) { - return true; - } - } else if (onnxDimension.getSize() == 0) { - return true; - } - } - return false; - } - @Override public String toString() { return "(" + valueType.id() + ")" + diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java deleted file mode 100644 index 2ef81e3f1fa..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java +++ /dev/null @@ -1,153 +0,0 @@ -package com.yahoo.vespa.model.ml; - -import com.fasterxml.jackson.core.JsonEncoding; -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.config.application.api.ApplicationFile; -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.io.IOUtils; -import com.yahoo.path.Path; -import com.yahoo.tensor.TensorType; - -import java.io.BufferedReader; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; -import java.util.Map; - -/** - * Defers to 'vespa-analyze-onnx-model' to determine the output type given - * a set of inputs. For situations with symbolic dimension sizes that can't - * easily be determined. - * - * @author lesters - */ -public class OnnxModelProbe { - - private static final String binary = "vespa-analyze-onnx-model"; - - static TensorType probeModel(ApplicationPackage app, Path modelPath, String outputName, Map<String, TensorType> inputTypes) { - TensorType outputType = TensorType.empty; - String contextKey = createContextKey(outputName, inputTypes); - - try { - // Check if output type has already been probed - outputType = readProbedOutputType(app, modelPath, contextKey); - - // Otherwise, run vespa-analyze-onnx-model if the model is available - if (outputType.equals(TensorType.empty) && app.getFile(modelPath).exists()) { - String jsonInput = createJsonInput(app.getFileReference(modelPath).getAbsolutePath(), inputTypes); - String jsonOutput = callVespaAnalyzeOnnxModel(jsonInput); - outputType = outputTypeFromJson(jsonOutput, outputName); - if ( ! outputType.equals(TensorType.empty)) { - writeProbedOutputType(app, modelPath, contextKey, outputType); - } - } - - } catch (IOException | InterruptedException e) { - e.printStackTrace(System.err); - } - - return outputType; - } - - private static String createContextKey(String onnxName, Map<String, TensorType> inputTypes) { - StringBuilder key = new StringBuilder().append(onnxName).append(":"); - inputTypes.entrySet().stream().sorted(Map.Entry.comparingByKey()) - .forEachOrdered(e -> key.append(e.getKey()).append(":").append(e.getValue()).append(",")); - return key.substring(0, key.length()-1); - } - - private static Path probedOutputTypesPath(Path path) { - String fileName = OnnxModelInfo.asValidIdentifier(path.getRelative()) + ".probed_output_types"; - return ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR.append(fileName); - } - - static void writeProbedOutputType(ApplicationPackage app, Path modelPath, String output, - Map<String, TensorType> inputTypes, TensorType type) throws IOException { - writeProbedOutputType(app, modelPath, createContextKey(output, inputTypes), type); - } - - private static void writeProbedOutputType(ApplicationPackage app, Path modelPath, - String contextKey, TensorType type) throws IOException { - String path = app.getFileReference(probedOutputTypesPath(modelPath)).getAbsolutePath(); - IOUtils.writeFile(path, contextKey + "\t" + type + "\n", true); - } - - private static TensorType readProbedOutputType(ApplicationPackage app, Path modelPath, - String contextKey) throws IOException { - ApplicationFile file = app.getFile(probedOutputTypesPath(modelPath)); - if ( ! file.exists()) { - return TensorType.empty; - } - try (BufferedReader reader = new BufferedReader(file.createReader())) { - String line; - while (null != (line = reader.readLine())) { - String[] parts = line.split("\t"); - String key = parts[0]; - if (key.equals(contextKey)) { - return TensorType.fromSpec(parts[1]); - } - } - } - return TensorType.empty; - } - - private static TensorType outputTypeFromJson(String json, String outputName) throws IOException { - ObjectMapper m = new ObjectMapper(); - JsonNode root = m.readTree(json); - if ( ! root.isObject() || ! root.has("outputs")) { - return TensorType.empty; - } - JsonNode outputs = root.get("outputs"); - if ( ! outputs.has(outputName)) { - return TensorType.empty; - } - return TensorType.fromSpec(outputs.get(outputName).asText()); - } - - private static String createJsonInput(String modelPath, Map<String, TensorType> inputTypes) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - JsonGenerator g = new JsonFactory().createGenerator(out, JsonEncoding.UTF8); - g.writeStartObject(); - g.writeStringField("model", modelPath); - g.writeObjectFieldStart("inputs"); - for (Map.Entry<String, TensorType> input : inputTypes.entrySet()) { - g.writeStringField(input.getKey(), input.getValue().toString()); - } - g.writeEndObject(); - g.writeEndObject(); - g.close(); - return out.toString(); - } - - private static String callVespaAnalyzeOnnxModel(String jsonInput) throws IOException, InterruptedException { - ProcessBuilder processBuilder = new ProcessBuilder(binary, "--probe-types"); - processBuilder.redirectErrorStream(true); - StringBuilder output = new StringBuilder(); - Process process = processBuilder.start(); - - // Write json array to process stdin - OutputStream os = process.getOutputStream(); - os.write(jsonInput.getBytes(StandardCharsets.UTF_8)); - os.close(); - - // Read output from stdout/stderr - InputStream inputStream = process.getInputStream(); - while (true) { - int b = inputStream.read(); - if (b == -1) break; - output.append((char)b); - } - int returnCode = process.waitFor(); - if (returnCode != 0) { - throw new IllegalArgumentException("Error from '" + binary + "'. Return code: " + returnCode + ". Output:\n" + output); - } - return output.toString(); - } - -} diff --git a/config-model/src/main/javacc/IntermediateParser.jj b/config-model/src/main/javacc/IntermediateParser.jj index 90260cb9895..7447eee5cec 100644 --- a/config-model/src/main/javacc/IntermediateParser.jj +++ b/config-model/src/main/javacc/IntermediateParser.jj @@ -1050,13 +1050,10 @@ void attribute(ParsedField field) : { <ATTRIBUTE> [name = identifier()] { - ParsedAttribute attr = new ParsedAttribute(name); + ParsedAttribute attr = field.attributeFor(name); } ( (<COLON> attributeSetting(attr)) | (lbrace() (attributeSetting(attr) (<NL>)*)* <RBRACE>) ) - { - field.addAttribute(attr); - } } /* pick up sorting in field block */ @@ -1164,7 +1161,12 @@ void summaryInDocument(ParsedDocumentSummary docsum) : } lbrace() (summaryItem(psf) (<NL>)*)* <RBRACE> { - docsum.addField(psf); + var old = docsum.addField(psf); + if (old != null) { + deployLogger.logApplicationPackage(Level.WARNING, "Summary field '" + psf.name() + + "' is defined twice in document-summary '" + + docsum.name() + "'"); + } } } @@ -1910,7 +1912,13 @@ void function(ParsedRankProfile profile) : { func.setExpression(expression); func.setInline(inline); - profile.addFunction(func); + var old = profile.addOrReplaceFunction(func); + if (old != null) { + /* TODO disallow this for Vespa 8 */ + deployLogger.logApplicationPackage(Level.WARNING, "Function '" + func.name() + + "' is defined twice in rank profile '" + + profile.name() + "'"); + } } } diff --git a/config-model/src/test/cfg/application/onnx_probe/files/create_dynamic_model.py b/config-model/src/test/cfg/application/onnx_probe/files/create_dynamic_model.py deleted file mode 100755 index b493e394ee4..00000000000 --- a/config-model/src/test/cfg/application/onnx_probe/files/create_dynamic_model.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -import onnx -import numpy as np -from onnx import helper, TensorProto - -INPUT_1 = helper.make_tensor_value_info('input1', TensorProto.FLOAT, ["batch", 2]) -INPUT_2 = helper.make_tensor_value_info('input2', TensorProto.FLOAT, ["batch", 2]) -OUTPUT = helper.make_tensor_value_info('out', TensorProto.FLOAT, ["batch", "dim1", "dim2"]) - -SHAPE = helper.make_tensor('shape', TensorProto.INT64, dims=[3], vals=np.array([1,2,2]).astype(np.int64)) - -nodes = [ - helper.make_node('Concat', ['input1', 'input2'], ['concat'], axis=1), - helper.make_node('Reshape', ['concat', 'shape'], ['out']), -] -graph_def = helper.make_graph(nodes, 'simple_scoring', [INPUT_1, INPUT_2], [OUTPUT], [SHAPE]) -model_def = helper.make_model(graph_def, producer_name='create_dynamic_model.py', opset_imports=[onnx.OperatorSetIdProto(version=12)]) -onnx.save(model_def, 'dynamic_model_2.onnx') diff --git a/config-model/src/test/cfg/application/onnx_probe/files/dynamic_model.onnx b/config-model/src/test/cfg/application/onnx_probe/files/dynamic_model.onnx deleted file mode 100644 index 28c600e2a09..00000000000 --- a/config-model/src/test/cfg/application/onnx_probe/files/dynamic_model.onnx +++ /dev/null @@ -1,21 +0,0 @@ -create_dynamic_model_2.py:Ö -- -input1 -input2concat"Concat* -axis - -concat -shapeout"Reshapesimple_scoring*:BshapeZ -input1 -
-batch -Z -input2 -
-batch -b& -out - -batch -dim1 -dim2B
\ No newline at end of file diff --git a/config-model/src/test/derived/advanced/index-info.cfg b/config-model/src/test/derived/advanced/index-info.cfg index c693597a75e..cf33b038700 100644 --- a/config-model/src/test/derived/advanced/index-info.cfg +++ b/config-model/src/test/derived/advanced/index-info.cfg @@ -51,6 +51,18 @@ indexinfo[].command[].indexname "debug" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "debug" indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "location.x" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "location.x" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "location.x" +indexinfo[].command[].command "type int" +indexinfo[].command[].indexname "location.y" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "location.y" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "location.y" +indexinfo[].command[].command "type int" indexinfo[].command[].indexname "location" indexinfo[].command[].command "default-position" indexinfo[].command[].indexname "location" diff --git a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg index 5434b0770f7..6429932ab0e 100644 --- a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg +++ b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg @@ -1,27 +1,27 @@ maxtermoccurrences 100 fieldmatchmaxlength 1000000 ilscript[].doctype "multiplesummaries" -ilscript[].docfield[0] "a" -ilscript[].docfield[1] "adynamic" -ilscript[].docfield[2] "abolded" -ilscript[].docfield[3] "b" -ilscript[].docfield[4] "c" -ilscript[].docfield[5] "d" -ilscript[].docfield[6] "e" -ilscript[].docfield[7] "f" -ilscript[].docfield[8] "g" -ilscript[].docfield[9] "h" -ilscript[].docfield[10] "loc" -ilscript[].docfield[11] "mytags" -ilscript[].content[0] "clear_state | guard { input loc | to_pos | zcurve | attribute loc_pos_zcurve; }" -ilscript[].content[1] "clear_state | guard { input a | tokenize normalize stem:\"BEST\" | summary abolded2 | summary aboldeddynamic | summary adynamic2 | attribute a; }" -ilscript[].content[2] "clear_state | guard { input adynamic | tokenize normalize stem:\"BEST\" | summary adynamic | attribute adynamic; }" -ilscript[].content[3] "clear_state | guard { input abolded | tokenize normalize stem:\"BEST\" | summary abolded | attribute abolded; }" -ilscript[].content[4] "clear_state | guard { input b | summary anotherb | summary b; }" -ilscript[].content[5] "clear_state | guard { input c | summary c | attribute c; }" -ilscript[].content[6] "clear_state | guard { input d | tokenize normalize stem:\"BEST\" | summary d; }" -ilscript[].content[7] "clear_state | guard { input e | tokenize normalize stem:\"BEST\" | summary dynamice | summary e; }" -ilscript[].content[8] "clear_state | guard { input f | summary f; }" -ilscript[].content[9] "clear_state | guard { input g | summary g; }" -ilscript[].content[10] "clear_state | guard { input h | summary h; }" -ilscript[].content[11] "clear_state | guard { input mytags | for_each { tokenize normalize stem:\"BEST\" } | index mytags; }" +ilscript[].docfield[] "a" +ilscript[].docfield[] "adynamic" +ilscript[].docfield[] "abolded" +ilscript[].docfield[] "b" +ilscript[].docfield[] "c" +ilscript[].docfield[] "d" +ilscript[].docfield[] "e" +ilscript[].docfield[] "f" +ilscript[].docfield[] "g" +ilscript[].docfield[] "h" +ilscript[].docfield[] "loc" +ilscript[].docfield[] "mytags" +ilscript[].content[] "clear_state | guard { input loc | to_pos | zcurve | attribute loc_pos_zcurve; }" +ilscript[].content[] "clear_state | guard { input a | tokenize normalize stem:\"BEST\" | summary abolded2 | summary aboldeddynamic | summary adynamic2 | attribute a; }" +ilscript[].content[] "clear_state | guard { input adynamic | tokenize normalize stem:\"BEST\" | summary adynamic | attribute adynamic; }" +ilscript[].content[] "clear_state | guard { input abolded | tokenize normalize stem:\"BEST\" | summary abolded | attribute abolded; }" +ilscript[].content[] "clear_state | guard { input b | summary anotherb | summary b; }" +ilscript[].content[] "clear_state | guard { input c | summary c | attribute c; }" +ilscript[].content[] "clear_state | guard { input d | tokenize normalize stem:\"BEST\" | summary d; }" +ilscript[].content[] "clear_state | guard { input e | tokenize normalize stem:\"BEST\" | summary dynamice | summary e; }" +ilscript[].content[] "clear_state | guard { input f | summary f; }" +ilscript[].content[] "clear_state | guard { input g | summary g; }" +ilscript[].content[] "clear_state | guard { input h | summary h; }" +ilscript[].content[] "clear_state | guard { input mytags | for_each { tokenize normalize stem:\"BEST\" } | index mytags; }" diff --git a/config-model/src/test/derived/multiplesummaries/index-info.cfg b/config-model/src/test/derived/multiplesummaries/index-info.cfg index d5002535761..6167dcbef02 100644 --- a/config-model/src/test/derived/multiplesummaries/index-info.cfg +++ b/config-model/src/test/derived/multiplesummaries/index-info.cfg @@ -111,6 +111,18 @@ indexinfo[].command[].indexname "dynamice" indexinfo[].command[].command "index" indexinfo[].command[].indexname "dynamice" indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "loc_pos.x" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "loc_pos.x" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "loc_pos.x" +indexinfo[].command[].command "type int" +indexinfo[].command[].indexname "loc_pos.y" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "loc_pos.y" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "loc_pos.y" +indexinfo[].command[].command "type int" indexinfo[].command[].indexname "loc_pos" indexinfo[].command[].command "default-position" indexinfo[].command[].indexname "loc_pos" diff --git a/config-model/src/test/derived/position_extra/index-info.cfg b/config-model/src/test/derived/position_extra/index-info.cfg index 45813140b1d..97521c14a6a 100644 --- a/config-model/src/test/derived/position_extra/index-info.cfg +++ b/config-model/src/test/derived/position_extra/index-info.cfg @@ -7,6 +7,18 @@ indexinfo[].command[].indexname "pos_str" indexinfo[].command[].command "index" indexinfo[].command[].indexname "pos_str" indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "pos_ext.x" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "pos_ext.x" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "pos_ext.x" +indexinfo[].command[].command "type int" +indexinfo[].command[].indexname "pos_ext.y" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "pos_ext.y" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "pos_ext.y" +indexinfo[].command[].command "type int" indexinfo[].command[].indexname "pos_ext" indexinfo[].command[].command "default-position" indexinfo[].command[].indexname "pos_ext" diff --git a/config-model/src/test/derived/rankexpression/macro.expression b/config-model/src/test/derived/rankingexpression/macro.expression index 054b025b2e7..054b025b2e7 100644 --- a/config-model/src/test/derived/rankexpression/macro.expression +++ b/config-model/src/test/derived/rankingexpression/macro.expression diff --git a/config-model/src/test/derived/rankexpression/overflow.expression b/config-model/src/test/derived/rankingexpression/overflow.expression index 4d3a6faeeb5..4d3a6faeeb5 100644 --- a/config-model/src/test/derived/rankexpression/overflow.expression +++ b/config-model/src/test/derived/rankingexpression/overflow.expression diff --git a/config-model/src/test/derived/rankexpression/rank-profiles.cfg b/config-model/src/test/derived/rankingexpression/rank-profiles.cfg index 6400870b0c4..cf59ae9c326 100644 --- a/config-model/src/test/derived/rankexpression/rank-profiles.cfg +++ b/config-model/src/test/derived/rankingexpression/rank-profiles.cfg @@ -363,3 +363,12 @@ rankprofile[].fef.property[].name "vespa.match.feature" rankprofile[].fef.property[].value "rankingExpression(mymul)" rankprofile[].fef.property[].name "vespa.type.attribute.t1" rankprofile[].fef.property[].value "tensor(m{},v[3])" +rankprofile[].name "withboolean" +rankprofile[].fef.property[].name "rankingExpression(mytrue).rankingScript" +rankprofile[].fef.property[].value "true" +rankprofile[].fef.property[].name "rankingExpression(myfalse).rankingScript" +rankprofile[].fef.property[].value "false" +rankprofile[].fef.property[].name "rankingExpression(mybooleanexpression).rankingScript" +rankprofile[].fef.property[].value "5.0" +rankprofile[].fef.property[].name "vespa.type.attribute.t1" +rankprofile[].fef.property[].value "tensor(m{},v[3])" diff --git a/config-model/src/test/derived/rankexpression/rankexpression.expression b/config-model/src/test/derived/rankingexpression/rankexpression.expression index d6cb73c829d..d6cb73c829d 100644 --- a/config-model/src/test/derived/rankexpression/rankexpression.expression +++ b/config-model/src/test/derived/rankingexpression/rankexpression.expression diff --git a/config-model/src/test/derived/rankexpression/rankexpression.sd b/config-model/src/test/derived/rankingexpression/rankexpression.sd index 73eb1cadadb..25d657eaf0b 100644 --- a/config-model/src/test/derived/rankexpression/rankexpression.sd +++ b/config-model/src/test/derived/rankingexpression/rankexpression.sd @@ -69,7 +69,8 @@ search rankexpression { rank-features: attribute(foo1).out attribute(bar1) rank-features { attribute(foo2).out attribute(bar2).out } rank-features { - attribute(foo3).out attribute(bar3).out } + attribute(foo3).out attribute(bar3).out + } rank-features { attribute(foo4).out attribute(bar4).out @@ -101,7 +102,8 @@ search rankexpression { summary-features: attribute(foo1).out attribute(bar1) summary-features { attribute(foo2).out attribute(bar2).out } summary-features { - attribute(foo3).out attribute(bar3).out } + attribute(foo3).out attribute(bar3).out + } summary-features { attribute(foo4).out attribute(bar4).out @@ -162,10 +164,10 @@ search rankexpression { second-phase { expression: fourtimessum(match,rankBoost) } - macro fourtimessum(var1, var2) { + function fourtimessum(var1, var2) { expression: 4*(var1+var2) } - macro myfeature() { + function myfeature() { expression { 70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness, 2) + 30 * pow(0 - fieldMatch(description).earliness, 2) @@ -187,21 +189,21 @@ search rankexpression { second-phase { expression: fourtimessum(match,match) + mysummaryfeature + myfeature } - macro fourtimessum(var1, var2) { + function fourtimessum(var1, var2) { expression: 4*(var1+var2) } - macro myfeature() { + function myfeature() { expression { 70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness, 2) + 30 * pow(0 - fieldMatch(description).earliness, 2) } } - macro mysummaryfeature() { + function mysummaryfeature() { expression { 70 * fieldMatch(title).completeness } } - macro mysummaryfeature2() { + function mysummaryfeature2() { expression { 71 * fieldMatch(title).completeness } @@ -213,7 +215,7 @@ search rankexpression { } rank-profile macros3 { - macro onlyusedinsummaryfeature() { + function onlyusedinsummaryfeature() { expression: 5 } summary-features { @@ -229,7 +231,7 @@ search rankexpression { } rank-profile macros-inherited inherits macros2 { - macro mysummaryfeature() { + function mysummaryfeature() { expression { 80 * fieldMatch(title).completeness } @@ -250,7 +252,7 @@ search rankexpression { } rank-profile macros-inherited3 inherits macros-inherited2 { - macro myfeature() { + function myfeature() { expression { 700 * fieldMatch(title).completeness } @@ -263,17 +265,17 @@ search rankexpression { } rank-profile macros-refering-macros { - macro m2() { + function m2() { expression: m1 * 67 } - macro m1() { + function m1() { expression { 700 * fieldMatch(title).completeness } } - macro m4() { + function m4() { expression: file:macro.expression } @@ -286,7 +288,7 @@ search rankexpression { } rank-profile macros-refering-macros-inherited inherits macros-refering-macros { - macro m3() { + function m3() { expression { if(isNan(attribute(nrtgmp))==1, 0.0, @@ -310,7 +312,7 @@ search rankexpression { } rank-profile macros-refering-macros-inherited-two-levels inherits macros-refering-macros-inherited { - macro m5() { + function m5() { expression { if(isNan(attribute(glmpfw))==1, m1, @@ -336,7 +338,8 @@ search rankexpression { match-features: attribute(foo1).out attribute(bar1) match-features { attribute(foo2).out attribute(bar2).out } match-features { - attribute(foo3).out attribute(bar3).out } + attribute(foo3).out attribute(bar3).out + } match-features { attribute(foo4) myplus @@ -346,4 +349,16 @@ search rankexpression { } } + rank-profile withboolean { + function mytrue() { + expression: true + } + function myfalse() { + expression: false + } + function mybooleanexpression() { + expression: true + if(false, 3.0, 4.0) + } + } + } diff --git a/config-model/src/test/derived/rankexpression/summary.cfg b/config-model/src/test/derived/rankingexpression/summary.cfg index aec076aa8fe..aec076aa8fe 100644 --- a/config-model/src/test/derived/rankexpression/summary.cfg +++ b/config-model/src/test/derived/rankingexpression/summary.cfg diff --git a/config-model/src/test/derived/rankexpression/summarymap.cfg b/config-model/src/test/derived/rankingexpression/summarymap.cfg index bd92b36548c..bd92b36548c 100644 --- a/config-model/src/test/derived/rankexpression/summarymap.cfg +++ b/config-model/src/test/derived/rankingexpression/summarymap.cfg diff --git a/config-model/src/test/derived/types/index-info.cfg b/config-model/src/test/derived/types/index-info.cfg index 2db4ead180b..7f43cd67a6b 100644 --- a/config-model/src/test/derived/types/index-info.cfg +++ b/config-model/src/test/derived/types/index-info.cfg @@ -96,7 +96,7 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "tagfield" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "tagfield" -indexinfo[].command[].command "type tag" +indexinfo[].command[].command "type WeightedSet<string>" indexinfo[].command[].indexname "structfield.s1" indexinfo[].command[].command "index" indexinfo[].command[].indexname "structfield.s1" @@ -704,4 +704,4 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "pst_sta_boldingoff_nomatch_tag_01" indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "pst_sta_boldingoff_nomatch_tag_01" -indexinfo[].command[].command "type tag" +indexinfo[].command[].command "type WeightedSet<string>" diff --git a/config-model/src/test/examples/attributesettings.sd b/config-model/src/test/examples/attributesettings.sd index 2b83f137b96..e7bb2062227 100644 --- a/config-model/src/test/examples/attributesettings.sd +++ b/config-model/src/test/examples/attributesettings.sd @@ -33,12 +33,17 @@ search attributesettings { indexing: attribute weightedset: remove-if-zero weightedset: create-if-nonexistent + attribute: fast-search + attribute: fast-access + attribute: paged } field f6 type weightedset<string> { weightedset: remove-if-zero indexing: attribute weightedset: create-if-nonexistent + attribute: enable-bit-vectors + attribute: enable-only-bit-vector } field f7 type weightedset<string> { diff --git a/config-model/src/test/examples/badparse.sd b/config-model/src/test/examples/badparse.sd new file mode 100755 index 00000000000..caeb1856e6b --- /dev/null +++ b/config-model/src/test/examples/badparse.sd @@ -0,0 +1,7 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search foo { + document foo { + something invalid here { + } + } +} diff --git a/config-model/src/test/examples/multiplesummaries.sd b/config-model/src/test/examples/multiplesummaries.sd index 07ab3c5a104..83e817718aa 100644 --- a/config-model/src/test/examples/multiplesummaries.sd +++ b/config-model/src/test/examples/multiplesummaries.sd @@ -28,6 +28,11 @@ search multiplesummaries { summary field3 type array<int> { } + # TODO disallow duplicates + summary field1 type weightedset<string> { + source: field1 + } + } } diff --git a/config-model/src/test/examples/structoutsideofdocument.sd b/config-model/src/test/examples/structoutsideofdocument.sd index 806351cd709..77bb3224816 100644 --- a/config-model/src/test/examples/structoutsideofdocument.sd +++ b/config-model/src/test/examples/structoutsideofdocument.sd @@ -1,7 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. search structoutsideofdocument { - # (will fail) + # (will fail with old parser) struct nalle { field foo type int {} @@ -13,3 +13,5 @@ search structoutsideofdocument { indexing: summary } } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java index 7b2ed2d7a7f..2a58618e11f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java @@ -74,6 +74,14 @@ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { assertWeightedSet(schema, "f8", true, false); assertWeightedSet(schema, "f9", false, true); assertWeightedSet(schema, "f10", false, true); + + assertAttrSettings(schema, "f4", false, false, false); + assertAttrSettings(schema, "f5", true, true, true); + assertAttrSettings(schema, "f6", false, false, false); + assertAttrSettings(schema, "f7", false, false, false); + assertAttrSettings(schema, "f8", false, false, false); + assertAttrSettings(schema, "f9", false, false, false); + assertAttrSettings(schema, "f10", false, false, false); } private void assertWeightedSet(Schema schema, String name, boolean createIfNonExistent, boolean removeIfZero) { @@ -82,13 +90,21 @@ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { Attribute a4 = f4.getAttributes().get(f4.getName()); assertEquals(Attribute.Type.STRING, a4.getType()); assertEquals(Attribute.CollectionType.WEIGHTEDSET, a4.getCollectionType()); - assertFalse(a4.isHuge()); - assertFalse(a4.isFastSearch()); - assertFalse(a4.isFastAccess()); assertEquals(a4.isRemoveIfZero(), removeIfZero); assertEquals(a4.isCreateIfNonExistent(), createIfNonExistent); } + private void assertAttrSettings(Schema schema, String name, boolean fastAccess, boolean fastSearch, boolean paged) { + SDField f4 = (SDField) schema.getDocument().getField(name); + assertEquals(1, f4.getAttributes().size()); + Attribute a4 = f4.getAttributes().get(f4.getName()); + assertEquals(Attribute.Type.STRING, a4.getType()); + assertEquals(Attribute.CollectionType.WEIGHTEDSET, a4.getCollectionType()); + assertEquals(a4.isFastSearch(), fastSearch); + assertEquals(a4.isFastAccess(), fastAccess); + assertEquals(a4.isPaged(), paged); + } + @Test public void requireThatFastAccessCanBeSet() throws IOException, ParseException { Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/attributesettings.sd"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java index 0fcc6e09b02..1bf07aff9d5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -12,8 +13,11 @@ import java.io.IOException; * @author bratseth */ public class MultipleSummariesTestCase extends AbstractSchemaTestCase { + @Test public void testArrayImporting() throws IOException, ParseException { - ApplicationBuilder.buildFromFile("src/test/examples/multiplesummaries.sd"); + var builder = new ApplicationBuilder(new TestProperties().setExperimentalSdParsing(true)); + builder.addSchemaFile("src/test/examples/multiplesummaries.sd"); + builder.build(true); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java index 94d25deb16a..e297beef026 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java @@ -192,14 +192,19 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { @Test public void testFunctionInliningWithReplacement() throws ParseException { + checkFunctionReplacement(false); + checkFunctionReplacement(true); + } + + public void checkFunctionReplacement(boolean useXPP) throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); MockDeployLogger deployLogger = new MockDeployLogger(); ApplicationBuilder builder = new ApplicationBuilder(MockApplicationPackage.createEmpty(), - new MockFileRegistry(), - deployLogger, - new TestProperties(), - rankProfileRegistry, - new QueryProfileRegistry()); + new MockFileRegistry(), + deployLogger, + new TestProperties().setExperimentalSdParsing(useXPP), + rankProfileRegistry, + new QueryProfileRegistry()); builder.addSchema( "search test {\n" + " document test { }\n" + @@ -219,8 +224,8 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { Schema s = builder.getSchema(); RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); assertEquals("foo(2)", test.getFirstPhaseRanking().getRoot().toString()); - assertTrue("Does not contain expected warning", deployLogger.contains("Function 'foo' replaces " + - "a previous function with the same name in rank profile 'test'")); + assertTrue("Does not contain expected warning", + deployLogger.contains("Function 'foo' is defined twice in rank profile 'test'")); } /** diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java index c27dd9dfdfb..32f82f44ad0 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java @@ -1,10 +1,13 @@ + // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.document.config.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.searchdefinition.derived.Deriver; import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Ignore; import org.junit.Test; import java.io.IOException; import static org.junit.Assert.fail; @@ -27,15 +30,19 @@ public class StructTestCase extends AbstractSchemaTestCase { try { ApplicationBuilder.buildFromFile("src/test/examples/badstruct.sd"); fail("Should throw exception."); - } catch (ParseException expected) { + } catch (IllegalArgumentException|ParseException expected) { + System.err.println("As expected, with message: "+expected.getMessage()); // success } } @Test + @Ignore public void testStructAndDocumentWithSameNames() { try { DocumenttypesConfig.Builder dt = Deriver.getDocumentTypesConfig("src/test/examples/structanddocumentwithsamenames.sd"); + // while the above line may work, the config generated will fail. + // See also NameCollisionTestCase. } catch (Exception e) { fail("Should not have thrown exception " + e); } @@ -46,7 +53,17 @@ public class StructTestCase extends AbstractSchemaTestCase { */ @Test(expected = IllegalArgumentException.class) public void testStructOutsideDocumentIllegal() throws IOException, ParseException { - ApplicationBuilder.buildFromFile("src/test/examples/structoutsideofdocument.sd"); + var builder = new ApplicationBuilder(new TestProperties().setExperimentalSdParsing(false)); + builder.addSchemaFile("src/test/examples/structoutsideofdocument.sd"); + } + + /** + * Declaring a struct before a document should work + */ + @Test + public void testStructOutsideDocumentLegal() throws IOException, ParseException { + var builder = new ApplicationBuilder(new TestProperties().setExperimentalSdParsing(true)); + builder.addSchemaFile("src/test/examples/structoutsideofdocument.sd"); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java index 456efdb08ae..ad4ff4be4b3 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java @@ -100,6 +100,7 @@ public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase { DeployLogger logger) throws IOException, ParseException { return assertCorrectDeriving(dirName, searchDefinitionName, new TestProperties(), logger); } + protected DerivedConfiguration assertCorrectDeriving(String dirName, TestProperties properties) throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AnnotationsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AnnotationsTestCase.java index b9f471f4cc7..65d6c15d3e2 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AnnotationsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AnnotationsTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -13,55 +14,74 @@ public class AnnotationsTestCase extends AbstractExportingTestCase { @Test public void requireThatStructRegistersIfOnlyUsedByAnnotation() throws IOException, ParseException { - assertCorrectDeriving("annotationsstruct"); + assertCorrectDeriving("annotationsstruct", + new TestProperties().setExperimentalSdParsing(true)); } @Test public void requireThatStructRegistersIfOnlyUsedAsArrayByAnnotation() throws IOException, ParseException { - assertCorrectDeriving("annotationsstructarray"); + assertCorrectDeriving("annotationsstructarray", + new TestProperties().setExperimentalSdParsing(true)); } @Test public void testSimpleAnnotationDeriving() throws IOException, ParseException { - assertCorrectDeriving("annotationssimple"); + assertCorrectDeriving("annotationssimple", + new TestProperties().setExperimentalSdParsing(true)); } @Test public void testAnnotationDerivingWithImplicitStruct() throws IOException, ParseException { - assertCorrectDeriving("annotationsimplicitstruct"); + assertCorrectDeriving("annotationsimplicitstruct", + new TestProperties().setExperimentalSdParsing(true)); } @Test public void testAnnotationDerivingInheritance() throws IOException, ParseException { - assertCorrectDeriving("annotationsinheritance"); + assertCorrectDeriving("annotationsinheritance", + new TestProperties().setExperimentalSdParsing(true)); } @Test public void testAnnotationDerivingInheritance2() throws IOException, ParseException { - assertCorrectDeriving("annotationsinheritance2"); + assertCorrectDeriving("annotationsinheritance2", + new TestProperties().setExperimentalSdParsing(true)); } @Test public void testSimpleReference() throws IOException, ParseException { - assertCorrectDeriving("annotationsreference"); + assertCorrectDeriving("annotationsreference", + new TestProperties().setExperimentalSdParsing(true)); } @Test public void testAdvancedReference() throws IOException, ParseException { - assertCorrectDeriving("annotationsreference2"); + assertCorrectDeriving("annotationsreference2", + new TestProperties().setExperimentalSdParsing(true)); } @Test public void testAnnotationsPolymorphy() throws IOException, ParseException { - assertCorrectDeriving("annotationspolymorphy"); + assertCorrectDeriving("annotationspolymorphy", + new TestProperties().setExperimentalSdParsing(true)); } /** * An annotation declared before document {} won't work, no doc type to add it to. */ @Test(expected = IllegalArgumentException.class) - public void testAnnotationOutsideOfDocumment() throws IOException, ParseException { - assertCorrectDeriving("annotationsoutsideofdocument"); + public void testAnnotationOutsideOfDocumentOld() throws IOException, ParseException { + assertCorrectDeriving("annotationsoutsideofdocument", + new TestProperties().setExperimentalSdParsing(false)); + } + + /** + * An annotation declared before document {} should work. + */ + @Test + public void testAnnotationOutsideOfDocumentNew() throws IOException, ParseException { + assertCorrectDeriving("annotationsoutsideofdocument", + new TestProperties().setExperimentalSdParsing(true)); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index 719d353f03a..412206d34ed 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -25,7 +25,7 @@ public class ExportingTestCase extends AbstractExportingTestCase { @Test public void testPositionArray() throws IOException, ParseException { assertCorrectDeriving("position_array", - new TestProperties().setUseV8GeoPositions(true)); + new TestProperties().setExperimentalSdParsing(true).setUseV8GeoPositions(true)); } @Test @@ -49,7 +49,7 @@ public class ExportingTestCase extends AbstractExportingTestCase { @Test public void testPositionSummary() throws IOException, ParseException { assertCorrectDeriving("position_summary", - new TestProperties().setUseV8GeoPositions(true)); + new TestProperties().setExperimentalSdParsing(true).setUseV8GeoPositions(true)); } @Test @@ -108,14 +108,14 @@ public class ExportingTestCase extends AbstractExportingTestCase { } @Test - public void testRankExpression() throws IOException, ParseException { - assertCorrectDeriving("rankexpression"); + public void testRankingExpression() throws IOException, ParseException { + assertCorrectDeriving("rankingexpression"); } @Test public void testAvoidRenamingRankingExpression() throws IOException, ParseException { assertCorrectDeriving("renamedfeatures", "foo", - new TestProperties().setAvoidRenamingSummaryFeatures(true), + new TestProperties().setExperimentalSdParsing(true).setAvoidRenamingSummaryFeatures(true), new TestableDeployLogger()); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java index 34354612d04..5dcb9e4ca01 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -18,7 +19,16 @@ public class ImportedFieldsTestCase extends AbstractExportingTestCase { @Test public void configs_for_imported_struct_fields_are_derived() throws IOException, ParseException { - assertCorrectDeriving("imported_struct_fields", "child", new TestableDeployLogger()); + assertCorrectDeriving("imported_struct_fields", "child", + new TestProperties().setExperimentalSdParsing(false), + new TestableDeployLogger()); + } + + @Test + public void configs_for_imported_struct_fields_are_derived_new() throws IOException, ParseException { + assertCorrectDeriving("imported_struct_fields", "child", + new TestProperties().setExperimentalSdParsing(true), + new TestableDeployLogger()); } @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultipleSummariesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultipleSummariesTestCase.java index 0e6d7b8442f..b13e88f5f1f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultipleSummariesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultipleSummariesTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -14,6 +15,13 @@ import java.io.IOException; public class MultipleSummariesTestCase extends AbstractExportingTestCase { @Test public void testMultipleSummaries() throws IOException, ParseException { - assertCorrectDeriving("multiplesummaries"); + assertCorrectDeriving("multiplesummaries", + new TestProperties().setExperimentalSdParsing(false)); + } + + @Test + public void testMultipleSummariesNew() throws IOException, ParseException { + assertCorrectDeriving("multiplesummaries", + new TestProperties().setExperimentalSdParsing(true)); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NameCollisionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NameCollisionTestCase.java index 5fce5c06943..70f6187be12 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NameCollisionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NameCollisionTestCase.java @@ -2,7 +2,12 @@ package com.yahoo.searchdefinition.derived; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.searchdefinition.ApplicationBuilder; + import org.junit.Test; +import static org.junit.Assert.assertThrows; /** * Verifies that a struct in a document type is preferred over another document type @@ -14,7 +19,21 @@ public class NameCollisionTestCase extends AbstractExportingTestCase { @Test public void testNameCollision() throws Exception { - assertCorrectDeriving("namecollision", "collisionstruct", new TestableDeployLogger()); + var ex = assertThrows(IllegalArgumentException.class, () -> { + assertCorrectDeriving("namecollision", "collisionstruct", + new TestProperties().setExperimentalSdParsing(false), + new TestableDeployLogger()); + var docman = DocumentTypeManager.fromFile("temp/namecollision/documentmanager.cfg"); + }); + System.err.println("MSG 1: "+ex.getClass()+" -> "+ex.getMessage()); + var ey = assertThrows(IllegalArgumentException.class, () -> { + assertCorrectDeriving("namecollision", "collisionstruct", + new TestProperties().setExperimentalSdParsing(true), + new TestableDeployLogger()); + var docman = DocumentTypeManager.fromFile("temp/namecollision/documentmanager.cfg"); + }); + System.err.println("MSG 2: "+ey.getMessage()); + } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/parser/ConvertIntermediateTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/parser/ConvertIntermediateTestCase.java index 264481cb3ec..127f5184c4c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/parser/ConvertIntermediateTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/parser/ConvertIntermediateTestCase.java @@ -26,7 +26,6 @@ public class ConvertIntermediateTestCase { var collection = new IntermediateCollection(); ParsedSchema schema = collection.addSchemaFromString(input); assertEquals("foo", schema.getDocument().name()); - collection.resolveInternalConnections(); var docMan = new DocumentTypeManager(); var converter = new ConvertSchemaCollection(collection, docMan); converter.convertTypes(); @@ -41,7 +40,6 @@ public class ConvertIntermediateTestCase { collection.addSchemaFromFile("src/test/derived/deriver/grandparent.sd"); collection.addSchemaFromFile("src/test/derived/deriver/parent.sd"); assertEquals(collection.getParsedSchemas().size(), 3); - collection.resolveInternalConnections(); var docMan = new DocumentTypeManager(); var converter = new ConvertSchemaCollection(collection, docMan); converter.convertTypes(); @@ -60,7 +58,6 @@ public class ConvertIntermediateTestCase { collection.addSchemaFromFile("src/test/converter/other.sd"); collection.addSchemaFromFile("src/test/converter/parent.sd"); collection.addSchemaFromFile("src/test/converter/grandparent.sd"); - collection.resolveInternalConnections(); var docMan = new DocumentTypeManager(); var converter = new ConvertSchemaCollection(collection, docMan); converter.convertTypes(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateCollectionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateCollectionTestCase.java index cd78f1faeb2..e29e4833856 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateCollectionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateCollectionTestCase.java @@ -36,6 +36,20 @@ public class IntermediateCollectionTestCase { } @Test + public void names_may_differ() throws Exception { + String input = joinLines + ("schema foo_search {", + " document foo {", + " }", + "}"); + var collection = new IntermediateCollection(); + ParsedSchema schema = collection.addSchemaFromString(input); + assertEquals("foo_search", schema.name()); + assertTrue(schema.hasDocument()); + assertEquals("foo", schema.getDocument().name()); + } + + @Test public void can_add_schema_files() throws Exception { var collection = new IntermediateCollection(); collection.addSchemaFromFile("src/test/derived/deriver/child.sd"); @@ -127,16 +141,16 @@ public class IntermediateCollectionTestCase { public void bad_parse_throws() throws Exception { var collection = new IntermediateCollection(); var ex = assertThrows(ParseException.class, () -> - collection.addSchemaFromFile("src/test/examples/structoutsideofdocument.sd")); - assertTrue(ex.getMessage().startsWith("Failed parsing schema from src/test/examples/structoutsideofdocument.sd: Encountered")); + collection.addSchemaFromFile("src/test/examples/badparse.sd")); + assertTrue(ex.getMessage().startsWith("Failed parsing schema from src/test/examples/badparse.sd: Encountered")); ex = assertThrows(ParseException.class, () -> - collection.addSchemaFromReader(readerOf("src/test/examples/structoutsideofdocument.sd"))); - assertTrue(ex.getMessage().startsWith("Failed parsing schema from src/test/examples/structoutsideofdocument.sd: Encountered")); + collection.addSchemaFromReader(readerOf("src/test/examples/badparse.sd"))); + assertTrue(ex.getMessage().startsWith("Failed parsing schema from src/test/examples/badparse.sd: Encountered")); collection.addSchemaFromFile("src/test/derived/rankprofilemodularity/test.sd"); collection.addRankProfileFile("test", "src/test/derived/rankprofilemodularity/test/outside_schema1.profile"); ex = assertThrows(ParseException.class, () -> - collection.addRankProfileFile("test", "src/test/examples/structoutsideofdocument.sd")); - assertTrue(ex.getMessage().startsWith("Failed parsing rank-profile from src/test/examples/structoutsideofdocument.sd: Encountered")); + collection.addRankProfileFile("test", "src/test/examples/badparse.sd")); + assertTrue(ex.getMessage().startsWith("Failed parsing rank-profile from src/test/examples/badparse.sd: Encountered")); } @Test @@ -167,7 +181,7 @@ public class IntermediateCollectionTestCase { assertEquals(collection.getParsedSchemas().size(), 3); var ex = assertThrows(IllegalArgumentException.class, () -> collection.resolveInternalConnections()); - assertTrue(ex.getMessage().startsWith("Inheritance cycle for schemas: ")); + assertTrue(ex.getMessage().startsWith("Inheritance/reference cycle for schemas: ")); } @Test @@ -180,7 +194,7 @@ public class IntermediateCollectionTestCase { var ex = assertThrows(IllegalArgumentException.class, () -> collection.resolveInternalConnections()); System.err.println("ex: "+ex.getMessage()); - assertTrue(ex.getMessage().startsWith("Inheritance cycle for documents: ")); + assertTrue(ex.getMessage().startsWith("Inheritance/reference cycle for documents: ")); } @Test @@ -194,4 +208,30 @@ public class IntermediateCollectionTestCase { assertEquals("document foo inherits from unavailable document bar", ex.getMessage()); } + @Test + public void can_detect_document_reference_cycle() throws Exception { + var collection = new IntermediateCollection(); + collection.addSchemaFromString("schema foo { document foo { field oneref type reference<bar> {} } }"); + collection.addSchemaFromString("schema bar { document bar { field tworef type reference<foo> {} } }"); + assertEquals(collection.getParsedSchemas().size(), 2); + var ex = assertThrows(IllegalArgumentException.class, () -> + collection.resolveInternalConnections()); + System.err.println("ex: "+ex.getMessage()); + assertTrue(ex.getMessage().startsWith("Inheritance/reference cycle for documents: ")); + } + + @Test + public void can_detect_cycles_with_reference() throws Exception { + var collection = new IntermediateCollection(); + collection.addSchemaFromString("schema foo { document foodoc inherits bardoc {} }"); + collection.addSchemaFromString("schema bar { document bardoc { field myref type reference<qux> { } } }"); + collection.addSchemaFromString("schema qux inherits foo { document qux inherits foodoc {} }"); + assertEquals(collection.getParsedSchemas().size(), 3); + var ex = assertThrows(IllegalArgumentException.class, () -> + collection.resolveInternalConnections()); + System.err.println("ex: "+ex.getMessage()); + assertTrue(ex.getMessage().startsWith("Inheritance/reference cycle for documents: ")); + } + + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java index 4e212ccd574..8bd04af8c54 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java @@ -22,8 +22,12 @@ public class IntermediateParserTestCase { var deployLogger = new BaseDeployLogger(); var modelProperties = new TestProperties(); var stream = new SimpleCharStream(input); - var parser = new IntermediateParser(stream, deployLogger, modelProperties); - return parser.schema(); + try { + var parser = new IntermediateParser(stream, deployLogger, modelProperties); + return parser.schema(); + } catch (ParseException pe) { + throw new ParseException(stream.formatException(pe.getMessage())); + } } ParsedSchema parseFile(String fileName) throws Exception { @@ -174,7 +178,7 @@ public class IntermediateParserTestCase { checkFileParses("src/test/derived/position_summary/position_summary.sd"); checkFileParses("src/test/derived/predicate_attribute/predicate_attribute.sd"); checkFileParses("src/test/derived/prefixexactattribute/prefixexactattribute.sd"); - checkFileParses("src/test/derived/rankexpression/rankexpression.sd"); + checkFileParses("src/test/derived/rankingexpression/rankexpression.sd"); checkFileParses("src/test/derived/rankprofileinheritance/child.sd"); checkFileParses("src/test/derived/rankprofileinheritance/parent1.sd"); checkFileParses("src/test/derived/rankprofileinheritance/parent2.sd"); @@ -208,9 +212,9 @@ public class IntermediateParserTestCase { checkFileParses("src/test/derived/uri_wset/uri_wset.sd"); checkFileParses("src/test/examples/arrays.sd"); checkFileParses("src/test/examples/arraysweightedsets.sd"); + checkFileParses("src/test/examples/attributeposition.sd"); checkFileParses("src/test/examples/attributesettings.sd"); checkFileParses("src/test/examples/attributesexactmatch.sd"); - checkFileParses("src/test/examples/badstruct.sd"); checkFileParses("src/test/examples/casing.sd"); checkFileParses("src/test/examples/comment.sd"); checkFileParses("src/test/examples/documentidinsummary.sd"); @@ -252,7 +256,6 @@ public class IntermediateParserTestCase { checkFileParses("src/test/examples/stemmingdefault.sd"); checkFileParses("src/test/examples/stemmingsetting.sd"); checkFileParses("src/test/examples/strange.sd"); - checkFileParses("src/test/examples/structanddocumentwithsamenames.sd"); checkFileParses("src/test/examples/struct.sd"); checkFileParses("src/test/examples/summaryfieldcollision.sd"); checkFileParses("src/test/examples/weightedset-summaryto.sd"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/OnnxModelProbeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/OnnxModelProbeTest.java deleted file mode 100644 index 6c4c919a229..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/OnnxModelProbeTest.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.ml; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.io.IOUtils; -import com.yahoo.path.Path; -import com.yahoo.tensor.TensorType; -import org.junit.Test; - -import java.io.IOException; -import java.util.Map; - -import static org.junit.Assert.assertEquals; - -public class OnnxModelProbeTest { - - @Test - public void testProbedOutputTypes() throws IOException { - - Path appDir = Path.fromString("src/test/cfg/application/onnx_probe"); - Path storedAppDir = appDir.append("copy"); - try { - FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir.toFile()); - Path modelPath = Path.fromString("files/dynamic_model.onnx"); - String output = "out"; - Map<String, TensorType> inputTypes = Map.of( - "input1", TensorType.fromSpec("tensor<float>(d0[1],d1[2])"), - "input2", TensorType.fromSpec("tensor<float>(d0[1],d1[2])")); - TensorType expected = TensorType.fromSpec("tensor<float>(d0[1],d1[2],d2[2])"); - - // Can't test model probing directly as 'vespa-analyze-onnx-model' is not available - TensorType outputType = OnnxModelProbe.probeModel(app, modelPath, output, inputTypes); - assertEquals(outputType, TensorType.empty); - - OnnxModelProbe.writeProbedOutputType(app, modelPath, output, inputTypes, expected); - - // Test loading from generated info - storedAppDir.toFile().mkdirs(); - IOUtils.copyDirectory(appDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), - storedAppDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - app = FilesApplicationPackage.fromFile(storedAppDir.toFile()); - outputType = OnnxModelProbe.probeModel(app, modelPath, output, inputTypes); - assertEquals(outputType, expected); - - } finally { - IOUtils.recursiveDeleteDir(appDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - IOUtils.recursiveDeleteDir(storedAppDir.toFile()); - } - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 6222e7b1788..dec5cbb7ee6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -210,6 +210,7 @@ public class ModelContextImpl implements ModelContext { private final boolean useQrserverServiceName; private final boolean avoidRenamingSummaryFeatures; private final boolean mergeGroupingResultInSearchInvoker; + private final boolean experimentalSdParsing; public FeatureFlags(FlagSource source, ApplicationId appId) { this.defaultTermwiseLimit = flagValue(source, appId, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -255,9 +256,10 @@ public class ModelContextImpl implements ModelContext { this.persistenceThrottlingWsResizeRate = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_RESIZE_RATE); this.persistenceThrottlingOfMergeFeedOps = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_OF_MERGE_FEED_OPS); this.inhibitDefaultMergesWhenGlobalMergesPending = flagValue(source, appId, Flags.INHIBIT_DEFAULT_MERGES_WHEN_GLOBAL_MERGES_PENDING); - this.useQrserverServiceName = flagValue(source, appId, Flags.USE_QRSERVER_SERVICE_NAME); + this.useQrserverServiceName = flagValue(source, appId, Flags.USE_QRSERVER_SERVICE_NAME); this.avoidRenamingSummaryFeatures = flagValue(source, appId, Flags.AVOID_RENAMING_SUMMARY_FEATURES); this.mergeGroupingResultInSearchInvoker = flagValue(source, appId, Flags.MERGE_GROUPING_RESULT_IN_SEARCH_INVOKER); + this.experimentalSdParsing = flagValue(source, appId, Flags.EXPERIMENTAL_SD_PARSING); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @@ -308,6 +310,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean useQrserverServiceName() { return useQrserverServiceName; } @Override public boolean avoidRenamingSummaryFeatures() { return avoidRenamingSummaryFeatures; } @Override public boolean mergeGroupingResultInSearchInvoker() { return mergeGroupingResultInSearchInvoker; } + @Override public boolean experimentalSdParsing() { return experimentalSdParsing; } private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/container-core/src/main/sh/find-pid b/container-core/src/main/sh/find-pid index 56c387387d4..4db6562db1b 100755 --- a/container-core/src/main/sh/find-pid +++ b/container-core/src/main/sh/find-pid @@ -87,7 +87,7 @@ if [ -f "$pid_file" ]; then parent_pid=$(cat "$pid_file") pid=$(pgrep --parent $parent_pid) else - status=$(vespa-sentinel-cmd list 2>/dev/null | grep --perl-regexp "($service state=|id=\"$service\")") + status=$(vespa-sentinel-cmd list 2>/dev/null | grep --perl-regexp "(^$service state=|id=\"$service\")") if [ -z "${status}" ]; then echo "No service with name or config id '${service}'" >&2 exit 1 diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java index 1db288d59dd..46dcaf17abc 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java @@ -2,12 +2,10 @@ package com.yahoo.container.jdisc.messagebus; import com.google.inject.Inject; -import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.component.AbstractComponent; import com.yahoo.container.jdisc.ContainerMbusConfig; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentUtil; -import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; @@ -70,24 +68,24 @@ public final class SessionCache extends AbstractComponent { @Inject public SessionCache(NetworkMultiplexerProvider nets, ContainerMbusConfig containerMbusConfig, - DocumentmanagerConfig documentmanagerConfig, + DocumentTypeManager documentTypeManager, LoadTypeConfig loadTypeConfig, MessagebusConfig messagebusConfig, DocumentProtocolPoliciesConfig policiesConfig, DistributionConfig distributionConfig) { - this(nets::net, containerMbusConfig, documentmanagerConfig, + this(nets::net, containerMbusConfig, documentTypeManager, loadTypeConfig, messagebusConfig, policiesConfig, distributionConfig); } public SessionCache(Supplier<NetworkMultiplexer> net, ContainerMbusConfig containerMbusConfig, - DocumentmanagerConfig documentmanagerConfig, + DocumentTypeManager documentTypeManager, LoadTypeConfig loadTypeConfig, MessagebusConfig messagebusConfig, DocumentProtocolPoliciesConfig policiesConfig, DistributionConfig distributionConfig) { this(net, containerMbusConfig, messagebusConfig, - new DocumentProtocol(new DocumentTypeManager(documentmanagerConfig), + new DocumentProtocol(documentTypeManager, new LoadTypeSet(loadTypeConfig), policiesConfig, distributionConfig)); diff --git a/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java b/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java index 8d8010c72dc..c509fb917fa 100644 --- a/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java +++ b/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java @@ -3,6 +3,7 @@ package com.yahoo.container.jdisc.messagebus; import com.yahoo.container.jdisc.ContainerMbusConfig; import com.yahoo.container.jdisc.config.SessionConfig; +import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.messagebus.MessagebusConfig; @@ -38,7 +39,7 @@ public class MbusClientProviderTest { private void testClient(SessionConfig config) { SessionCache cache = new SessionCache(() -> NetworkMultiplexer.dedicated(new NullNetwork()), new ContainerMbusConfig.Builder().build(), - new DocumentmanagerConfig.Builder().build(), + new DocumentTypeManager(new DocumentmanagerConfig.Builder().build()), new LoadTypeConfig.Builder().build(), new MessagebusConfig.Builder().build(), new DocumentProtocolPoliciesConfig.Builder().build(), diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java index c52e36986be..d7592d4f97a 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java @@ -274,7 +274,6 @@ class ResultBuilder { return count; } - HitList newHitList(int listIdx, int tag, HitsAggregationResult execResult) { HitList hitList = new HitList(transform.getLabel(tag)); List<Hit> hits = execResult.getHits(); diff --git a/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java b/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java index 97ae9e8b113..b89994d63da 100644 --- a/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java +++ b/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java @@ -6,7 +6,7 @@ import java.util.Set; /** * A hit which holds information on error conditions in a result. - * En error hit maintains a main error - the main error of the result. + * An error hit maintains a main error - the main error of the result. * * @author bratseth */ diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AccessControlService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AccessControlService.java index a08319055ff..1dd6eb543ef 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AccessControlService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AccessControlService.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.athenz; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.athenz.api.AthenzRoleInformation; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.api.OAuthCredentials; @@ -16,8 +17,9 @@ import java.util.Collection; */ public interface AccessControlService { boolean approveDataPlaneAccess(AthenzUser user, Instant expiry); - boolean approveSshAccess(TenantName tenantName, Instant expiry, OAuthCredentials oAuthCredentials); + boolean decideSshAccess(TenantName tenantName, Instant expiry, OAuthCredentials oAuthCredentials, boolean approve); boolean requestSshAccess(TenantName tenantName); - boolean hasPendingAccessRequests(TenantName tenantName); + AthenzRoleInformation getAccessRoleInformation(TenantName tenantName); + void setPreapprovedAccess(TenantName tenantName, boolean preapproved); Collection<AthenzUser> listMembers(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzAccessControlService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzAccessControlService.java index 0568678219e..415a087d990 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzAccessControlService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzAccessControlService.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzGroup; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzRole; +import com.yahoo.vespa.athenz.api.AthenzRoleInformation; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.api.OAuthCredentials; import com.yahoo.vespa.athenz.client.zms.ZmsClient; @@ -26,11 +27,13 @@ public class AthenzAccessControlService implements AccessControlService { private final AthenzRole dataPlaneAccessRole; private final AthenzGroup vespaTeam; private final ZmsClient vespaZmsClient; //TODO: Merge ZMS clients + private final AthenzInstanceSynchronizer athenzInstanceSynchronizer; - public AthenzAccessControlService(ZmsClient zmsClient, AthenzDomain domain, ZmsClient vespaZmsClient) { + public AthenzAccessControlService(ZmsClient zmsClient, AthenzDomain domain, ZmsClient vespaZmsClient, AthenzInstanceSynchronizer athenzInstanceSynchronizer) { this.zmsClient = zmsClient; this.vespaZmsClient = vespaZmsClient; + this.athenzInstanceSynchronizer = athenzInstanceSynchronizer; this.dataPlaneAccessRole = new AthenzRole(domain, DATAPLANE_ACCESS_ROLENAME); this.vespaTeam = new AthenzGroup(domain, ALLOWED_OPERATOR_GROUPNAME); } @@ -43,7 +46,7 @@ public class AthenzAccessControlService implements AccessControlService { } Map<AthenzIdentity, String> users = zmsClient.listPendingRoleApprovals(dataPlaneAccessRole); if (users.containsKey(user)) { - zmsClient.approvePendingRoleMembership(dataPlaneAccessRole, user, expiry, Optional.empty(), Optional.empty()); + zmsClient.decidePendingRoleMembership(dataPlaneAccessRole, user, expiry, Optional.empty(), Optional.empty(), true); return true; } return false; @@ -62,19 +65,19 @@ public class AthenzAccessControlService implements AccessControlService { * @return Whether the ssh access role has any pending role membership requests */ @Override - public boolean hasPendingAccessRequests(TenantName tenantName) { + public AthenzRoleInformation getAccessRoleInformation(TenantName tenantName) { var role = sshRole(tenantName); if (!vespaZmsClient.listRoles(role.domain()).contains(role)) - return false; - var pendingApprovals = vespaZmsClient.listPendingRoleApprovals(role); - return pendingApprovals.containsKey(vespaTeam); + vespaZmsClient.createRole(role, Map.of()); + + return vespaZmsClient.getFullRoleInformation(role); } /** * @return true if access has been granted - false if already member */ @Override - public boolean approveSshAccess(TenantName tenantName, Instant expiry, OAuthCredentials oAuthCredentials) { + public boolean decideSshAccess(TenantName tenantName, Instant expiry, OAuthCredentials oAuthCredentials, boolean approve) { var role = sshRole(tenantName); if (!vespaZmsClient.listRoles(role.domain()).contains(role)) @@ -83,10 +86,13 @@ public class AthenzAccessControlService implements AccessControlService { if (vespaZmsClient.getMembership(role, vespaTeam)) return false; - if (!hasPendingAccessRequests(tenantName)) { - vespaZmsClient.addRoleMember(role, vespaTeam, Optional.empty()); - } - vespaZmsClient.approvePendingRoleMembership(role, vespaTeam, expiry, Optional.empty(), Optional.of(oAuthCredentials)); + var roleInformation = vespaZmsClient.getFullRoleInformation(role); + if (roleInformation.getPendingRequest().isEmpty()) + return false; + var reason = roleInformation.getPendingRequest().get().getReason(); + + vespaZmsClient.decidePendingRoleMembership(role, vespaTeam, expiry, Optional.of(reason), Optional.of(oAuthCredentials), approve); + athenzInstanceSynchronizer.synchronizeInstances(tenantName); return true; } @@ -107,6 +113,16 @@ public class AthenzAccessControlService implements AccessControlService { return true; } + public void setPreapprovedAccess(TenantName tenantName, boolean preapprovedAccess) { + var role = sshRole(tenantName); + + var attributes = Map.<String, Object>of( + "selfServe", !preapprovedAccess, + "reviewEnabled", !preapprovedAccess + ); + vespaZmsClient.createRole(role, attributes); + } + private AthenzRole sshRole(TenantName tenantName) { return new AthenzRole(getOrCreateTenantDomain(tenantName), "ssh_access"); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzInstanceSynchronizer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzInstanceSynchronizer.java new file mode 100644 index 00000000000..3b9166d4363 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzInstanceSynchronizer.java @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athenz; + +import com.yahoo.config.provision.TenantName; + +/** + * @author olaa + * + * Responsible for synchronizing misc roles and their pending memberships between separate Athenz instances + */ +public interface AthenzInstanceSynchronizer { + + void synchronizeInstances(TenantName tenant); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzInstanceSynchronizerMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzInstanceSynchronizerMock.java new file mode 100644 index 00000000000..1f0403a0b44 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzInstanceSynchronizerMock.java @@ -0,0 +1,12 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athenz; + +import com.yahoo.config.provision.TenantName; + +/** + * @author olaa + */ +public class AthenzInstanceSynchronizerMock implements AthenzInstanceSynchronizer { + @Override + public void synchronizeInstances(TenantName tenant) {} +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/MockAccessControlService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/MockAccessControlService.java index b8106450705..c14ca2bdc80 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/MockAccessControlService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/MockAccessControlService.java @@ -3,12 +3,16 @@ package com.yahoo.vespa.hosted.controller.api.integration.athenz; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzRoleInformation; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.api.OAuthCredentials; import java.time.Instant; import java.util.Collection; import java.util.HashSet; +import java.util.List; +import java.util.Optional; import java.util.Set; public class MockAccessControlService implements AccessControlService { @@ -31,7 +35,7 @@ public class MockAccessControlService implements AccessControlService { } @Override - public boolean approveSshAccess(TenantName tenantName, Instant expiry, OAuthCredentials oAuthCredentials) { + public boolean decideSshAccess(TenantName tenantName, Instant expiry, OAuthCredentials oAuthCredentials, boolean approve) { return false; } @@ -41,8 +45,13 @@ public class MockAccessControlService implements AccessControlService { } @Override - public boolean hasPendingAccessRequests(TenantName tenantName) { - return false; + public AthenzRoleInformation getAccessRoleInformation(TenantName tenantName) { + return new AthenzRoleInformation(new AthenzDomain("test-domain"), "tenant-role", false, false, Optional.empty(), List.of()); + } + + @Override + public void setPreapprovedAccess(TenantName tenantName, boolean preapproved) { + } public void addPendingMember(AthenzUser user) { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java index 38b2a36a348..5f567e8b84a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPolicy; import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; +import com.yahoo.vespa.athenz.api.AthenzRoleInformation; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.OAuthCredentials; import com.yahoo.vespa.athenz.client.zms.RoleAction; @@ -201,7 +202,7 @@ public class ZmsClientMock implements ZmsClient { } @Override - public void approvePendingRoleMembership(AthenzRole athenzRole, AthenzIdentity athenzIdentity, Instant expiry, Optional<String> reason, Optional<OAuthCredentials> oAuthCredentials) { + public void decidePendingRoleMembership(AthenzRole athenzRole, AthenzIdentity athenzIdentity, Instant expiry, Optional<String> reason, Optional<OAuthCredentials> oAuthCredentials, boolean approve) { } @Override @@ -256,6 +257,11 @@ public class ZmsClientMock implements ZmsClient { public void createSubdomain(AthenzDomain parent, String name) {} @Override + public AthenzRoleInformation getFullRoleInformation(AthenzRole role) { + return new AthenzRoleInformation(role.domain(), role.roleName(), true, true, Optional.empty(), List.of()); + } + + @Override public void close() {} private static AthenzDomain getTenantDomain(AthenzResourceName resource) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 225ffacb1ad..30f416747e0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -20,7 +20,6 @@ import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.security.PublicKey; import java.time.Instant; import java.util.ArrayDeque; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -226,6 +225,12 @@ public class Application { .map(Instance::quotaUsage).reduce(QuotaUsage::add).orElse(QuotaUsage.none); } + /** Returns the total quota usage for manual deployments for this application */ + public QuotaUsage manualQuotaUsage() { + return instances().values().stream() + .map(Instance::manualQuotaUsage).reduce(QuotaUsage::add).orElse(QuotaUsage.none); + } + /** Returns the total quota usage for this application, excluding one specific deployment (and temporary deployments) */ public QuotaUsage quotaUsage(ApplicationId application, ZoneId zone) { return instances().values().stream() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java index cc68d47666d..e5ab13d2127 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java @@ -200,6 +200,13 @@ public class Instance { .map(Deployment::quota).reduce(QuotaUsage::add).orElse(QuotaUsage.none); } + /** Returns the total quota usage for manual deployments for this instance **/ + public QuotaUsage manualQuotaUsage() { + return deployments.values().stream() + .filter(d -> d.zone().environment().isManuallyDeployed()) + .map(Deployment::quota).reduce(QuotaUsage::add).orElse(QuotaUsage.none); + } + /** Returns the total quota usage for this instance, excluding one specific deployment (and temporary deployments) */ public QuotaUsage quotaUsageExcluding(ApplicationId application, ZoneId zone) { return deployments.values().stream() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculator.java index 481f57e13f1..30ec0e202a2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculator.java @@ -69,19 +69,23 @@ public class DeploymentQuotaCalculator { private static Quota probablyEnoughForAll(Quota tenantQuota, List<Application> tenantApps, ApplicationId application, DeploymentSpec deploymentSpec) { - TenantAndApplicationId deployingApp = TenantAndApplicationId.from(application); + TenantAndApplicationId deployingAppId = TenantAndApplicationId.from(application); var usageOutsideApplication = tenantApps.stream() - .filter(app -> !app.id().equals(deployingApp)) + .filter(app -> !app.id().equals(deployingAppId)) .map(Application::quotaUsage).reduce(QuotaUsage::add).orElse(QuotaUsage.none); + QuotaUsage manualQuotaUsage = tenantApps.stream() + .filter(app -> app.id().equals(deployingAppId)).findFirst() + .map(Application::manualQuotaUsage).orElse(QuotaUsage.none); + long productionDeployments = Math.max(1, deploymentSpec.instances().stream() .flatMap(instance -> instance.zones().stream()) .filter(zone -> zone.environment().isProduction()) .count()); return tenantQuota.withBudget( - tenantQuota.subtractUsage(usageOutsideApplication.rate()) + tenantQuota.subtractUsage(usageOutsideApplication.rate() + manualQuotaUsage.rate()) .budget().get().divide(BigDecimal.valueOf(productionDeployments), 5, RoundingMode.HALF_UP)); // 1/1000th of a cent should be accurate enough } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 217bb9e1444..2ece93ba23f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -240,7 +240,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/notifications")) return notifications(request, Optional.ofNullable(request.getProperty("tenant")), true); if (path.matches("/application/v4/tenant")) return tenants(request); if (path.matches("/application/v4/tenant/{tenant}")) return tenant(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/access/request/ssh")) return accessRequests(path.get("tenant"), request); + if (path.matches("/application/v4/tenant/{tenant}/access/request/operator")) return accessRequests(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/info")) return tenantInfo(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/notifications")) return notifications(request, Optional.of(path.get("tenant")), false); if (path.matches("/application/v4/tenant/{tenant}/secret-store/{name}/validate")) return validateSecretStore(path.get("tenant"), path.get("name"), request); @@ -292,8 +292,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private HttpResponse handlePUT(Path path, HttpRequest request) { if (path.matches("/application/v4/tenant/{tenant}")) return updateTenant(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/access/request/ssh")) return requestSshAccess(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/access/approve/ssh")) return approveAccessRequest(path.get("tenant"), request); + if (path.matches("/application/v4/tenant/{tenant}/access/request/operator")) return requestSshAccess(path.get("tenant"), request); + if (path.matches("/application/v4/tenant/{tenant}/access/approve/operator")) return approveAccessRequest(path.get("tenant"), request); + if (path.matches("/application/v4/tenant/{tenant}/access/preapprove/operator")) return addPreapprovedAccess(path.get("tenant")); if (path.matches("/application/v4/tenant/{tenant}/info")) return updateTenantInfo(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/archive-access")) return allowArchiveAccess(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/secret-store/{name}")) return addSecretStore(path.get("tenant"), path.get("name"), request); @@ -341,6 +342,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private HttpResponse handleDELETE(Path path, HttpRequest request) { if (path.matches("/application/v4/tenant/{tenant}")) return deleteTenant(path.get("tenant"), request); + if (path.matches("/application/v4/tenant/{tenant}/access/preapprove/operator")) return removePreapprovedAccess(path.get("tenant")); if (path.matches("/application/v4/tenant/{tenant}/key")) return removeDeveloperKey(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/archive-access")) return removeArchiveAccess(path.get("tenant")); if (path.matches("/application/v4/tenant/{tenant}/secret-store/{name}")) return deleteSecretStore(path.get("tenant"), path.get("name"), request); @@ -414,9 +416,27 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud) return ErrorResponse.badRequest("Can only see access requests for cloud tenants"); - var pendingRequests = controller.serviceRegistry().accessControlService().hasPendingAccessRequests(TenantName.from(tenantName)); + var accessControlService = controller.serviceRegistry().accessControlService(); + var accessRoleInformation = accessControlService.getAccessRoleInformation(TenantName.from(tenantName)); + var preapprovedAccess = !accessRoleInformation.isSelfServe() && !accessRoleInformation.isReviewEnabled(); var slime = new Slime(); - slime.setObject().setBool("hasPendingRequests", pendingRequests); + var cursor = slime.setObject(); + cursor.setBool("preapprovedAccess", preapprovedAccess); + accessRoleInformation.getPendingRequest() + .ifPresent(membershipRequest -> { + var requestCursor = cursor.setObject("pendingRequest"); + requestCursor.setString("requestTime", membershipRequest.getCreationTime()); + requestCursor.setString("reason", membershipRequest.getReason()); + }); + var auditLogCursor = cursor.setArray("auditLog"); + accessRoleInformation.getAuditLog() + .forEach(auditLogEntry -> { + var entryCursor = auditLogCursor.addObject(); + entryCursor.setString("created", auditLogEntry.getCreationTime()); + entryCursor.setString("approver", auditLogEntry.getApprover()); + entryCursor.setString("reason", auditLogEntry.getReason()); + entryCursor.setString("status", auditLogEntry.getAction()); + }); return new SlimeJsonResponse(slime); } @@ -443,8 +463,27 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { var expiry = inspector.field("expiry").valid() ? Instant.ofEpochMilli(inspector.field("expiry").asLong()) : Instant.now().plus(1, ChronoUnit.DAYS); + var approve = inspector.field("approve").asBool(); + + controller.serviceRegistry().accessControlService().decideSshAccess(tenant, expiry, OAuthCredentials.fromAuth0RequestContext(request.getJDiscRequest().context()), approve); + return new MessageResponse("OK"); + } + + private HttpResponse addPreapprovedAccess(String tenantName) { + return setPreapprovedAccess(tenantName, true); + } + + private HttpResponse removePreapprovedAccess(String tenantName) { + return setPreapprovedAccess(tenantName, false); + } + + private HttpResponse setPreapprovedAccess(String tenantName, boolean preapprovedAccess) { + var tenant = TenantName.from(tenantName); + + if (controller.tenants().require(tenant).type() != Tenant.Type.cloud) + return ErrorResponse.badRequest("Can only set access privel for cloud tenants"); - controller.serviceRegistry().accessControlService().approveSshAccess(tenant, expiry, OAuthCredentials.fromAuth0RequestContext(request.getJDiscRequest().context())); + controller.serviceRegistry().accessControlService().setPreapprovedAccess(tenant, preapprovedAccess); return new MessageResponse("OK"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java index 2751b1697db..974df695d07 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java @@ -6,6 +6,8 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Instance; @@ -25,6 +27,7 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; +import java.util.TreeSet; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -56,6 +59,32 @@ public class DeploymentQuotaCalculatorTest { } @Test + public void quota_is_divided_among_prod_and_manual_instances() { + + var existing_dev_deployment = new Application(TenantAndApplicationId.from(ApplicationId.defaultId()), Instant.EPOCH, DeploymentSpec.empty, ValidationOverrides.empty, + Optional.empty(), Optional.empty(), Optional.empty(), OptionalInt.empty(), new ApplicationMetrics(1, 1), Set.of(), OptionalLong.empty(), + Optional.empty(), new TreeSet<>(), List.of(new Instance(ApplicationId.defaultId()).withNewDeployment( + ZoneId.from(Environment.dev, RegionName.defaultName()), ApplicationVersion.unknown, Version.emptyVersion, Instant.EPOCH, Map.of(), + QuotaUsage.create(0.53d)))); + + Quota calculated = DeploymentQuotaCalculator.calculate(Quota.unlimited().withBudget(2), List.of(existing_dev_deployment), ApplicationId.defaultId(), ZoneId.defaultId(), + DeploymentSpec.fromXml( + "<deployment version='1.0'>\n" + + " <instance id='default'> \n" + + " <test />\n" + + " <staging />\n" + + " <prod>\n" + + " <region active=\"true\">us-east-1</region>\n" + + " <region active=\"false\">us-west-1</region>\n" + + " <region active=\"true\">us-north-1</region>\n" + + " <region active=\"true\">us-south-1</region>\n" + + " </prod>\n" + + " </instance>\n" + + "</deployment>")); + assertEquals((2d - 0.53d) / 4d, calculated.budget().orElseThrow().doubleValue(), 1e-5); + } + + @Test public void unlimited_quota_remains_unlimited() { Quota calculated = DeploymentQuotaCalculator.calculate(Quota.unlimited(), List.of(), ApplicationId.defaultId(), ZoneId.defaultId(), DeploymentSpec.empty); assertTrue(calculated.isUnlimited()); diff --git a/docproc/src/main/java/com/yahoo/docproc/Processing.java b/docproc/src/main/java/com/yahoo/docproc/Processing.java index 47ebb6a7988..834d63c5a86 100644 --- a/docproc/src/main/java/com/yahoo/docproc/Processing.java +++ b/docproc/src/main/java/com/yahoo/docproc/Processing.java @@ -140,10 +140,8 @@ public class Processing { } /** - * @deprecated Use TBD instead + * @deprecated This method will be removed without replacement in Vespa 8. */ - // TODO: used to: processing.setDocprocServiceRegistry(this.documentProcessingHandler.getDocprocServiceRegistry()); - // from Processor and LoggingRequestHandler @Deprecated(forRemoval = true, since="7") @SuppressWarnings("removal") // TODO Vespa 8: remove public void setDocprocServiceRegistry(ComponentRegistry<DocprocService> docprocServiceRegistry) { @@ -166,9 +164,9 @@ public class Processing { * if #getServiceName returns a name that is not registered in {@link com.yahoo.docproc.DocprocService}. * * @return the service processing this, or null if unknown. - * @deprecated Use TBD instead + * @deprecated Formerly used to retrieve the {@link com.yahoo.document.DocumentTypeManager}, + * which can now be directly injected via your component constructor. */ - // TODO: used to getService().getDocumentTypeManager() in subclasses of DocumentProcessor @Deprecated(forRemoval = true, since="7") @SuppressWarnings("removal") // TODO Vespa 8: remove public DocprocService getService() { diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java index a949ec5ecf3..3b31b0447a3 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java @@ -102,7 +102,7 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { ComponentRegistry<AbstractConcreteDocumentFactory> docFactoryRegistry, ChainsConfig chainsConfig, SchemamappingConfig mappingConfig, - DocumentmanagerConfig docManConfig, + DocumentTypeManager documentTypeManager, DocprocConfig docprocConfig, ContainerDocumentConfig containerDocConfig, Metric metric) { @@ -110,7 +110,7 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { documentProcessorComponentRegistry, docFactoryRegistry, new DocumentProcessingHandlerParameters() .setMaxNumThreads(docprocConfig.numthreads()) - .setDocumentTypeManager(new DocumentTypeManager(docManConfig)) + .setDocumentTypeManager(documentTypeManager) .setChainsModel(buildFromConfig(chainsConfig)).setSchemaMap(configureMapping(mappingConfig)) .setMetric(metric) .setContainerDocumentConfig(containerDocConfig)); diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java index e8a2e214776..33cd6647ede 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java @@ -105,8 +105,10 @@ class ProcessingFactory { Processing processing = new Processing(); processing.addDocumentOperation(documentOperation); processing.setServiceName(serviceName); - // TODO Vespa 8: Use TBD instead, this method will be removed + + // TODO Vespa 8: Remove statement (registry will be removed from Processing) processing.setDocprocServiceRegistry(docprocServiceComponentRegistry); + processing.setVariable("route", message.getRoute()); processing.setVariable("timeout", message.getTimeRemaining()); return processing; diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/IndexingProcessor.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/IndexingProcessor.java index 15ccc17e318..7b553383daf 100644 --- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/IndexingProcessor.java +++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/IndexingProcessor.java @@ -52,11 +52,11 @@ public class IndexingProcessor extends DocumentProcessor { } @Inject - public IndexingProcessor(DocumentmanagerConfig documentmanagerConfig, + public IndexingProcessor(DocumentTypeManager documentTypeManager, IlscriptsConfig ilscriptsConfig, Linguistics linguistics, Embedder embedder) { - docTypeMgr = DocumentTypeManagerConfigurer.configureNewManager(documentmanagerConfig); + docTypeMgr = documentTypeManager; scriptMgr = new ScriptManager(docTypeMgr, ilscriptsConfig, linguistics, embedder); adapterFactory = new SimpleAdapterFactory(new ExpressionSelector()); } diff --git a/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java b/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java index ac1d3f21997..13f9ea1a8c8 100644 --- a/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java +++ b/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java @@ -7,6 +7,7 @@ import com.yahoo.document.Document; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentOperation; import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentUpdate; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.document.datatypes.StringFieldValue; @@ -124,7 +125,7 @@ public class IndexingProcessorTestCase { } private static IndexingProcessor newProcessor(String configId) { - return new IndexingProcessor(ConfigGetter.getConfig(DocumentmanagerConfig.class, configId), + return new IndexingProcessor(new DocumentTypeManager(ConfigGetter.getConfig(DocumentmanagerConfig.class, configId)), ConfigGetter.getConfig(IlscriptsConfig.class, configId), new SimpleLinguistics(), Embedder.throwsOnUse); diff --git a/document/src/main/java/com/yahoo/document/DataType.java b/document/src/main/java/com/yahoo/document/DataType.java index bb1777954a3..697536a8ac0 100644 --- a/document/src/main/java/com/yahoo/document/DataType.java +++ b/document/src/main/java/com/yahoo/document/DataType.java @@ -52,6 +52,7 @@ public abstract class DataType extends Identifiable implements Serializable, Com public final static DocumentType DOCUMENT = new DocumentType("document"); public final static PrimitiveDataType URI = new PrimitiveDataType("uri", 10, UriFieldValue.class, new UriFieldValue.Factory()); public final static NumericDataType BYTE = new NumericDataType("byte", 16, ByteFieldValue.class, ByteFieldValue.getFactory()); + final static int TAG_ID = 18; public final static PrimitiveDataType PREDICATE = new PrimitiveDataType("predicate", 20, PredicateFieldValue.class, PredicateFieldValue.getFactory()); public final static int tensorDataTypeCode = 21; // All TensorDataType instances have id=21 but carries additional type information serialized separately // ADDITIONAL parametrized types added at runtime: map, struct, array, weighted set, annotation reference, tensor diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index be2c182426e..9325e374daa 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -61,8 +61,13 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub public ConfigSubscriber configure(String configId) { ConfigSubscriber subscriber = new ConfigSubscriber(); - subscriber.subscribe(this, DocumentmanagerConfig.class, configId); - return subscriber; + try { + subscriber.subscribe(this, DocumentmanagerConfig.class, configId); + return subscriber; + } catch (RuntimeException e) { + subscriber.close(); + throw e; + } } /** One-shot configuration; should be called on a newly constructed manager */ @@ -264,7 +269,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub var old = configMap.put(id, dataTypeConfig); if (old != null) { throw new IllegalArgumentException - ("Multiple configs for id "+id+" first: "+old+" second: "+dataTypeConfig); + ("Multiple configs for id "+id+" first:\n"+old+"\nsecond:\n"+dataTypeConfig); } } } diff --git a/document/src/main/java/com/yahoo/document/WeightedSetDataType.java b/document/src/main/java/com/yahoo/document/WeightedSetDataType.java index 35dd13efb0b..b21f059bd7d 100644 --- a/document/src/main/java/com/yahoo/document/WeightedSetDataType.java +++ b/document/src/main/java/com/yahoo/document/WeightedSetDataType.java @@ -24,25 +24,33 @@ public class WeightedSetDataType extends CollectionDataType { public WeightedSetDataType(DataType nestedType, boolean createIfNonExistent, boolean removeIfZero) { this(nestedType, createIfNonExistent, removeIfZero, 0); - if ((nestedType == STRING) && createIfNonExistent && removeIfZero) { // the tag type definition - setId(18); - } else { - setId(getName().toLowerCase().hashCode()); - } } public WeightedSetDataType(DataType nestedType, boolean createIfNonExistent, boolean removeIfZero, int id) { super(createName(nestedType, createIfNonExistent, removeIfZero), id, nestedType); this.createIfNonExistent = createIfNonExistent; this.removeIfZero = removeIfZero; + if (id == 0) { + if ((nestedType == STRING) && createIfNonExistent && removeIfZero) { // the tag type definition + setId(TAG_ID); + } else { + setId(getName().toLowerCase().hashCode()); + } + } + int code = getId(); + if ((code >= 0) && (code <= DataType.lastPredefinedDataTypeId()) && (code != TAG_ID)) { + throw new IllegalArgumentException("Cannot create a weighted set datatype with code " + code); + } } + /* + * @deprecated // TODO remove on Vespa 8 + * Do not use - use one of the constructors above. + * Note: ignores typeName argument. + */ + @Deprecated public WeightedSetDataType(String typeName, int code, DataType nestedType, boolean createIfNonExistent, boolean removeIfZero) { - super(typeName != null ? createName(nestedType, createIfNonExistent, removeIfZero) : null, code, nestedType); - if ((code >= 0) && (code <= DataType.lastPredefinedDataTypeId()) && (code != 18)) // 18 == DataType.TAG.getId() is not yet initialized - throw new IllegalArgumentException("Cannot create a weighted set datatype with code " + code); - this.createIfNonExistent = createIfNonExistent; - this.removeIfZero = removeIfZero; + this(nestedType, createIfNonExistent, removeIfZero, code); } @Override diff --git a/document/src/tests/arrayfieldvaluetest.cpp b/document/src/tests/arrayfieldvaluetest.cpp index a8ca9d22916..375ce05a4e6 100644 --- a/document/src/tests/arrayfieldvaluetest.cpp +++ b/document/src/tests/arrayfieldvaluetest.cpp @@ -3,7 +3,6 @@ #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/vespalib/objects/nbostream.h> -#include <vespa/document/util/bytebuffer.h> #include <vespa/document/repo/documenttyperepo.h> #include <gtest/gtest.h> #include <gmock/gmock.h> @@ -103,21 +102,14 @@ TEST(ArrayFieldValueTest, testArray) ArrayFieldValue::UP valuePtr(value2.clone()); EXPECT_EQ(value, *valuePtr); - // Iterating + // Iterating const ArrayFieldValue& constVal(value); - for(ArrayFieldValue::const_iterator it = constVal.begin(); - it != constVal.end(); ++it) - { - const FieldValue& fval1(*it); - (void) fval1; - EXPECT_EQ((uint32_t) IntFieldValue::classId, - it->getClass().id()); + for(const FieldValue & fval1 : constVal) { + EXPECT_EQ((uint32_t) IntFieldValue::classId, fval1.getClass().id()); } value2 = value; - for(ArrayFieldValue::iterator it = value2.begin(); it != value2.end(); ++it) - { - (*it).assign(IntFieldValue(7)); - it->assign(IntFieldValue(7)); + for(size_t i(0); i < value2.size(); i++) { + value2[i].assign(IntFieldValue(7)); } EXPECT_TRUE(value != value2); EXPECT_TRUE(value2.contains(IntFieldValue(7))); diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp index f40aec41e16..d2fa969daa9 100644 --- a/document/src/tests/documentselectparsertest.cpp +++ b/document/src/tests/documentselectparsertest.cpp @@ -1,5 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/document/test/fieldvalue_helpers.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/update/assignvalueupdate.h> @@ -152,18 +153,18 @@ DocumentSelectParserTest::createDocs() // Add some arrays and structs to doc 1 { StructFieldValue sval(_doc.back()->getField("mystruct").getDataType()); - sval.set("key", 14); - sval.set("value", "structval"); + sval.setValue("key", IntFieldValue::make(14)); + sval.setValue("value", StringFieldValue::make("structval")); _doc.back()->setValue("mystruct", sval); ArrayFieldValue aval(_doc.back()->getField("structarray").getDataType()); { StructFieldValue sval1(aval.getNestedType()); - sval1.set("key", 15); - sval1.set("value", "structval1"); + sval1.setValue("key", IntFieldValue::make(15)); + sval1.setValue("value", StringFieldValue::make("structval1")); StructFieldValue sval2(aval.getNestedType()); - sval2.set("key", 16); - sval2.set("value", "structval2"); + sval2.setValue("key", IntFieldValue::make(16)); + sval2.setValue("value", StringFieldValue::make("structval2")); aval.add(sval1); aval.add(sval2); } @@ -181,11 +182,11 @@ DocumentSelectParserTest::createDocs() ArrayFieldValue abval(_doc.back()->getField("structarray").getDataType()); { StructFieldValue sval1(aval.getNestedType()); - sval1.set("key", 17); - sval1.set("value", "structval3"); + sval1.setValue("key", IntFieldValue::make(17)); + sval1.setValue("value", StringFieldValue::make("structval3")); StructFieldValue sval2(aval.getNestedType()); - sval2.set("key", 18); - sval2.set("value", "structval4"); + sval2.setValue("key", IntFieldValue::make(18)); + sval2.setValue("value", StringFieldValue::make("structval4")); abval.add(sval1); abval.add(sval2); } @@ -193,17 +194,15 @@ DocumentSelectParserTest::createDocs() amval.put(StringFieldValue("bar"), abval); _doc.back()->setValue("structarrmap", amval); - WeightedSetFieldValue wsval( - _doc.back()->getField("stringweightedset").getDataType()); - wsval.add("foo"); - wsval.add("val1"); - wsval.add("val2"); - wsval.add("val3"); - wsval.add("val4"); + WeightedSetFieldValue wsval(_doc.back()->getField("stringweightedset").getDataType()); + WSetHelper(wsval).add("foo"); + WSetHelper(wsval).add("val1"); + WSetHelper(wsval).add("val2"); + WSetHelper(wsval).add("val3"); + WSetHelper(wsval).add("val4"); _doc.back()->setValue("stringweightedset", wsval); - WeightedSetFieldValue wsbytes( - _doc.back()->getField("byteweightedset").getDataType()); + WeightedSetFieldValue wsbytes(_doc.back()->getField("byteweightedset").getDataType()); wsbytes.add(ByteFieldValue(5)); wsbytes.add(ByteFieldValue(75)); wsbytes.add(ByteFieldValue(static_cast<int8_t>(255))); @@ -211,14 +210,12 @@ DocumentSelectParserTest::createDocs() _doc.back()->setValue("byteweightedset", wsbytes); } - _doc.push_back(createDoc( - "testdoctype1", "id:myspace:testdoctype1:n=1234:footype1", 15, 1.0, "some", "some", 0)); // DOC 2 + _doc.push_back(createDoc("testdoctype1", "id:myspace:testdoctype1:n=1234:footype1", 15, 1.0, "some", "some", 0)); // DOC 2 // Add empty struct and array { StructFieldValue sval(_doc.back()->getField("mystruct").getDataType()); _doc.back()->setValue("mystruct", sval); - ArrayFieldValue aval( - _doc.back()->getField("structarray").getDataType()); + ArrayFieldValue aval(_doc.back()->getField("structarray").getDataType()); _doc.back()->setValue("structarray", aval); } _doc.push_back(createDoc("testdoctype1", "id:myspace:testdoctype1:g=yahoo:bar", 14, 2.4, "Yet", "\xE4\xB8\xBA\xE4\xBB\x80", 0)); // DOC 3 diff --git a/document/src/tests/documenttestcase.cpp b/document/src/tests/documenttestcase.cpp index fde702f2080..c852e219faa 100644 --- a/document/src/tests/documenttestcase.cpp +++ b/document/src/tests/documenttestcase.cpp @@ -1,5 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/document/test/fieldvalue_helpers.h> #include <vespa/document/base/testdocman.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/datatype/mapdatatype.h> @@ -349,11 +350,12 @@ TEST(DocumentTest, testModifyDocument) structmap1.put(StringFieldValue("test"), l2s1); l1s1.setValue(structmapF, structmap1); - WeightedSetFieldValue wset1(wset); + WeightedSetFieldValue wwset1(wset); + WSetHelper wset1(wwset1); wset1.add("foo"); wset1.add("bar"); wset1.add("zoo"); - l1s1.setValue(wsetF, wset1); + l1s1.setValue(wsetF, wwset1); WeightedSetFieldValue wset2(structwset); wset2.add(l2s1); @@ -705,19 +707,19 @@ TEST(DocumentTest,testReadSerializedAllVersions) // Create a memory instance of document { Document doc(*docType, DocumentId("id:ns:serializetest::http://test.doc.id/")); - doc.set("intfield", 5); - doc.set("floatfield", -9.23); - doc.set("stringfield", "This is a string."); - doc.set("longfield", static_cast<int64_t>(398420092938472983LL)); - doc.set("doublefield", 98374532.398820); - doc.set("bytefield", -2); - doc.setValue("rawfield", RawFieldValue("RAW DATA", 8)); + doc.setValue("intfield", IntFieldValue::make(5)); + doc.setValue("floatfield", FloatFieldValue::make(-9.23)); + doc.setValue("stringfield", StringFieldValue::make("This is a string.")); + doc.setValue("longfield", LongFieldValue::make(static_cast<int64_t>(398420092938472983LL))); + doc.setValue("doublefield", DoubleFieldValue::make(98374532.398820)); + doc.setValue("bytefield", ByteFieldValue::make(-2)); + doc.setValue("rawfield", std::make_unique<RawFieldValue>("RAW DATA", 8)); Document docInDoc(*docInDocType, DocumentId("id:ns:docindoc::http://doc.in.doc/")); - docInDoc.set("stringindocfield", "Elvis is dead"); + docInDoc.setValue("stringindocfield", StringFieldValue::make("Elvis is dead")); doc.setValue("docfield", docInDoc); ArrayFieldValue floatArray(*arrayOfFloatDataType); - floatArray.add(1.0); - floatArray.add(2.0); + CollectionHelper(floatArray).add(1.0); + CollectionHelper(floatArray).add(2.0); doc.setValue("arrayoffloatfield", floatArray); WeightedSetFieldValue weightedSet(*weightedSetDataType); weightedSet.add(StringFieldValue("Weighted 0"), 50); @@ -828,14 +830,14 @@ TEST(DocumentTest, testGenerateSerializedFile) DocumentTypeRepo repo(readDocumenttypesConfig(file_name)); Document doc(*repo.getDocumentType("serializetest"), DocumentId("id:ns:serializetest::http://test.doc.id/")); - doc.set("intfield", 5); - doc.set("floatfield", -9.23); - doc.set("stringfield", "This is a string."); - doc.set("longfield", (int64_t) 398420092938472983ll); - doc.set("doublefield", 98374532.398820); - doc.set("urifield", "http://this.is.a.test/"); - doc.set("bytefield", -2); - doc.set("rawfield", "RAW DATA"); + doc.setValue("intfield", IntFieldValue::make(5)); + doc.setValue("floatfield", FloatFieldValue::make(-9.23)); + doc.setValue("stringfield", StringFieldValue::make("This is a string.")); + doc.setValue("longfield", LongFieldValue::make((int64_t) 398420092938472983ll)); + doc.setValue("doublefield", DoubleFieldValue::make(98374532.398820)); + doc.setValue("urifield", StringFieldValue::make("http://this.is.a.test/")); + doc.setValue("bytefield", ByteFieldValue::make(-2)); + doc.setValue("rawfield", std::make_unique<RawFieldValue>("RAW DATA")); const DocumentType *docindoc_type = repo.getDocumentType("docindoc"); EXPECT_TRUE(docindoc_type); @@ -949,7 +951,7 @@ TEST(DocumentTest, testHasChanged) Document doc2(test_repo.getTypeRepo(), buf); EXPECT_TRUE(!doc2.hasChanged()); - doc2.set("headerval", 13); + doc2.setValue("headerval", IntFieldValue::make(13)); EXPECT_TRUE(doc2.hasChanged()); } // Overwriting a value in doc tags us changed. @@ -957,7 +959,7 @@ TEST(DocumentTest, testHasChanged) buf.rp(0); Document doc2(test_repo.getTypeRepo(), buf); - doc2.set("hstringval", "bla bla bla bla bla"); + doc2.setValue("hstringval", StringFieldValue::make("bla bla bla bla bla")); EXPECT_TRUE(doc2.hasChanged()); } // Clearing value tags us changed. diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index 8a9aef0bde4..26819699db4 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -1,5 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/document/test/fieldvalue_helpers.h> #include <vespa/document/base/testdocman.h> #include <vespa/document/base/exceptions.h> #include <vespa/document/datatype/tensor_data_type.h> @@ -138,8 +139,8 @@ TEST(DocumentUpdateTest, testSimpleUsage) // Create a test document Document doc(*docType, DocumentId("id:ns:test::1")); - doc.set("bytef", 0); - doc.set("intf", 5); + doc.setValue("bytef", ByteFieldValue::make(0)); + doc.setValue("intf", IntFieldValue::make(5)); ArrayFieldValue array(*arrayType); array.add(IntFieldValue(3)); array.add(IntFieldValue(7)); @@ -1287,9 +1288,9 @@ TEST(DocumentUpdateTest, array_element_update_applies_to_specified_element) ArrayUpdateFixture f; ArrayFieldValue array_value(f.array_field.getDataType()); - array_value.add("foo"); - array_value.add("baz"); - array_value.add("blarg"); + CollectionHelper(array_value).add("foo"); + CollectionHelper(array_value).add("baz"); + CollectionHelper(array_value).add("blarg"); f.doc->setValue(f.array_field, array_value); f.update->applyTo(*f.doc); @@ -1306,7 +1307,7 @@ TEST(DocumentUpdateTest, array_element_update_for_invalid_index_is_ignored) ArrayUpdateFixture f; ArrayFieldValue array_value(f.array_field.getDataType()); - array_value.add("jerry"); + CollectionHelper(array_value).add("jerry"); f.doc->setValue(f.array_field, array_value); f.update->applyTo(*f.doc); // MapValueUpdate for index 1, which does not exist diff --git a/document/src/tests/fieldpathupdatetestcase.cpp b/document/src/tests/fieldpathupdatetestcase.cpp index 3ebf3699763..3d41a5bcacd 100644 --- a/document/src/tests/fieldpathupdatetestcase.cpp +++ b/document/src/tests/fieldpathupdatetestcase.cpp @@ -1,4 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/document/test/fieldvalue_helpers.h> #include <vespa/document/base/testdocman.h> #include <vespa/document/fieldvalue/iteratorhandler.h> #include <vespa/document/fieldvalue/intfieldvalue.h> @@ -128,11 +130,12 @@ createTestDocument(const DocumentTypeRepo &repo) structmap1.put(StringFieldValue("test"), l2s1); l1s1.setValue("structmap", structmap1); - WeightedSetFieldValue wset1(*wset); + WeightedSetFieldValue wwset1(*wset); + WSetHelper wset1(wwset1); wset1.add("foo"); wset1.add("bar"); wset1.add("zoo"); - l1s1.setValue("wset", wset1); + l1s1.setValue("wset", wwset1); WeightedSetFieldValue wset2(*structwset); wset2.add(l2s1); diff --git a/document/src/tests/fieldvalue/fieldvalue_test.cpp b/document/src/tests/fieldvalue/fieldvalue_test.cpp index b70fd0d18a8..d8712768000 100644 --- a/document/src/tests/fieldvalue/fieldvalue_test.cpp +++ b/document/src/tests/fieldvalue/fieldvalue_test.cpp @@ -1,15 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for fieldvalue. -#include <vespa/log/log.h> -LOG_SETUP("fieldvalue_test"); - #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/fieldvalue/longfieldvalue.h> #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/log/log.h> +LOG_SETUP("fieldvalue_test"); + using namespace document; namespace { @@ -18,14 +18,6 @@ TEST("require that StringFieldValue can be assigned primitives") { StringFieldValue val; val = "foo"; EXPECT_EQUAL("foo", val.getValue()); - val = 1; - EXPECT_EQUAL("1", val.getValue()); - val = static_cast<int64_t>(2); - EXPECT_EQUAL("2", val.getValue()); - val = 3.0f; - EXPECT_EQUAL("3", val.getValue()); - val = 4.0; - EXPECT_EQUAL("4", val.getValue()); } TEST("require that FieldValues does not change their storage size.") { diff --git a/document/src/tests/primitivefieldvaluetest.cpp b/document/src/tests/primitivefieldvaluetest.cpp index 7c734544f27..ea78baa4ee6 100644 --- a/document/src/tests/primitivefieldvaluetest.cpp +++ b/document/src/tests/primitivefieldvaluetest.cpp @@ -3,7 +3,6 @@ #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/vespalib/objects/nbostream.h> -#include <vespa/document/util/bytebuffer.h> #include <vespa/document/repo/documenttyperepo.h> #include <limits> #include <gtest/gtest.h> @@ -32,7 +31,7 @@ void deserialize(nbostream & stream, T &value) { const Type& medium2, const Type& largest) { try{ - // Less + // Less EXPECT_TRUE(!(smallest < smallest)); EXPECT_TRUE(smallest < medium1); EXPECT_TRUE(smallest < medium2); @@ -203,31 +202,6 @@ TEST(PrimitiveFieldValueTest, testRaw) value.getValueRef().size()) == 0); } -#define ASSERT_FAILED_CONV(getter, totype, floating) \ -{ \ - totype toType; \ - FieldValue::UP copy(value.clone()); \ - try{ \ - getter; \ - std::ostringstream ost; \ - ost << "Conversion unexpectedly worked from max value of " \ - << *value.getDataType() << " to " << *toType.getDataType(); \ - FAIL() << ost.str(); \ - } catch (std::exception& e) { \ - EXPECT_EQ( \ - std::string("bad numeric conversion: positive overflow"), \ - std::string(e.what())); \ - } \ - /* Verify that we can convert to smaller type if value is within \ - range. Only tests integer to integer. No floating point. */ \ - if (!floating) { \ - totype::Number maxV = std::numeric_limits<totype::Number>::max(); \ - value.setValue((Number) maxV); \ - getter; \ - } \ - value.assign(*copy); \ -} - namespace { template<typename Numeric> @@ -258,25 +232,21 @@ namespace { // representation can keep the value. if (floatingPoint || sizeof(Number) > sizeof(unsigned char)) { // No longer throws. This is guarded on the perimeter by java code. - // ASSERT_FAILED_CONV(value.getAsByte(), ByteFieldValue, floatingPoint); } else { EXPECT_EQ((char) maxValue, value.getAsByte()); } if (floatingPoint || sizeof(Number) > sizeof(int32_t)) { // No longer throws. This is guarded on the perimeter by java code. - // ASSERT_FAILED_CONV(value.getAsInt(), IntFieldValue, floatingPoint); } else { EXPECT_EQ((int32_t) maxValue, value.getAsInt()); } if (floatingPoint || sizeof(Number) > sizeof(int64_t)) { // No longer throws. This is guarded on the perimeter by java code. - // ASSERT_FAILED_CONV(value.getAsLong(), LongFieldValue, floatingPoint); } else { EXPECT_EQ((int64_t) maxValue, value.getAsLong()); } if (floatingPoint && sizeof(Number) > sizeof(float)) { // No longer throws. This is guarded on the perimeter by java code. - // ASSERT_FAILED_CONV(value.getAsFloat(), FloatFieldValue, true); } else { EXPECT_EQ((float) maxValue, value.getAsFloat()); } @@ -303,30 +273,15 @@ TEST(PrimitiveFieldValueTest, testBool) v = BoolFieldValue(true); EXPECT_TRUE(v.getValue()); - v = 0; - EXPECT_TRUE( ! v.getValue()); - v = 1; - EXPECT_TRUE(v.getValue()); - - v = INT64_C(0); - EXPECT_TRUE( ! v.getValue()); - v = INT64_C(1); - EXPECT_TRUE(v.getValue()); - - v = 0.0f; - EXPECT_TRUE( ! v.getValue()); - v = 1.0f; - EXPECT_TRUE(v.getValue()); - - v = 0.0; - EXPECT_TRUE( ! v.getValue()); - v = 1.0; + v.setValue(false); + EXPECT_FALSE(v.getValue()); + v.setValue(true); EXPECT_TRUE(v.getValue()); v = vespalib::stringref("true"); EXPECT_TRUE(v.getValue()); v = vespalib::stringref("something not true"); - EXPECT_TRUE( ! v.getValue()); + EXPECT_FALSE(v.getValue()); } TEST(PrimitiveFieldValueTest, testNumerics) diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp index d85d5240b64..33e8c521e09 100644 --- a/document/src/tests/serialization/vespadocumentserializer_test.cpp +++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for vespadocumentserializer. +#include <vespa/document/test/fieldvalue_helpers.h> #include <vespa/document/annotation/annotation.h> #include <vespa/document/annotation/span.h> #include <vespa/document/annotation/spantree.h> @@ -345,7 +346,7 @@ void checkArrayFieldValue(SizeType value_count) { ArrayDataType array_type(*DataType::INT); ArrayFieldValue value(array_type); for (uint32_t i = 0; i < value_count; ++i) { - value.add(static_cast<int32_t>(i)); + CollectionHelper(value).add(static_cast<int32_t>(i)); } nbostream stream; diff --git a/document/src/tests/weightedsetfieldvaluetest.cpp b/document/src/tests/weightedsetfieldvaluetest.cpp index 325267e9da6..61f727120d1 100644 --- a/document/src/tests/weightedsetfieldvaluetest.cpp +++ b/document/src/tests/weightedsetfieldvaluetest.cpp @@ -1,5 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/document/test/fieldvalue_helpers.h> #include <vespa/document/fieldvalue/weightedsetfieldvalue.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/longfieldvalue.h> @@ -255,7 +256,8 @@ TEST(WeightedSetFieldValueTest, testWeightedSet) WeightedSetDataType mytype2(*DataType::STRING, true, true); EXPECT_EQ(*DataType::TAG, static_cast<DataType &>(mytype2)); - WeightedSetFieldValue val1(mytype1); + WeightedSetFieldValue wsval1(mytype1); + WSetHelper val1(wsval1); val1.add("foo", 4); try{ val1.increment("bar", 2); @@ -272,25 +274,26 @@ TEST(WeightedSetFieldValueTest, testWeightedSet) val1.decrement("foo", 3); EXPECT_EQ(7, val1.get("foo")); val1.decrement("foo", 7); - EXPECT_TRUE(val1.contains("foo")); + EXPECT_TRUE(CollectionHelper(wsval1).contains("foo")); - WeightedSetFieldValue val2(mytype2); + WeightedSetFieldValue wsval2(mytype2); + WSetHelper val2(wsval2); val2.add("foo", 4); val2.increment("bar", 2); EXPECT_EQ(2, val2.get("bar")); val2.decrement("bar", 4); EXPECT_EQ(-2, val2.get("bar")); val2.increment("bar", 2); - EXPECT_TRUE(!val2.contains("bar")); + EXPECT_TRUE(!CollectionHelper(wsval2).contains("bar")); val2.decrement("foo", 4); - EXPECT_TRUE(!val2.contains("foo")); + EXPECT_TRUE(!CollectionHelper(wsval2).contains("foo")); val2.decrement("foo", 4); EXPECT_EQ(-4, val2.get("foo")); val2.add("foo", 0); - EXPECT_TRUE(!val2.contains("foo")); + EXPECT_TRUE(!CollectionHelper(wsval2).contains("foo")); } } @@ -301,12 +304,12 @@ TEST(WeightedSetFieldValueTest, testAddIgnoreZeroWeight) WeightedSetFieldValue ws(wsetType); ws.addIgnoreZeroWeight(StringFieldValue("yarn"), 0); - EXPECT_TRUE(ws.contains("yarn")); - EXPECT_EQ(0, ws.get("yarn")); + EXPECT_TRUE(CollectionHelper(ws).contains("yarn")); + EXPECT_EQ(0, WSetHelper(ws).get("yarn")); ws.addIgnoreZeroWeight(StringFieldValue("flarn"), 1); - EXPECT_TRUE(ws.contains("flarn")); - EXPECT_EQ(1, ws.get("flarn")); + EXPECT_TRUE(CollectionHelper(ws).contains("flarn")); + EXPECT_EQ(1, WSetHelper(ws).get("flarn")); } } // document diff --git a/document/src/vespa/document/datatype/primitivedatatype.cpp b/document/src/vespa/document/datatype/primitivedatatype.cpp index 08e873c7de2..d5af0ca6885 100644 --- a/document/src/vespa/document/datatype/primitivedatatype.cpp +++ b/document/src/vespa/document/datatype/primitivedatatype.cpp @@ -56,16 +56,16 @@ FieldValue::UP PrimitiveDataType::createFieldValue() const { switch (getId()) { - case T_INT: return std::make_unique<IntFieldValue>(); - case T_SHORT: return std::make_unique<ShortFieldValue>(); - case T_FLOAT: return std::make_unique<FloatFieldValue>(); - case T_URI: return std::make_unique<StringFieldValue>(); - case T_STRING: return std::make_unique<StringFieldValue>(); + case T_INT: return IntFieldValue::make(); + case T_SHORT: return ShortFieldValue::make(); + case T_FLOAT: return FloatFieldValue::make(); + case T_URI: return StringFieldValue::make(); + case T_STRING: return StringFieldValue::make(); case T_RAW: return std::make_unique<RawFieldValue>(); - case T_LONG: return std::make_unique<LongFieldValue>(); - case T_DOUBLE: return std::make_unique<DoubleFieldValue>(); - case T_BOOL: return std::make_unique<BoolFieldValue>(); - case T_BYTE: return std::make_unique<ByteFieldValue>(); + case T_LONG: return LongFieldValue::make(); + case T_DOUBLE: return DoubleFieldValue::make(); + case T_BOOL: return BoolFieldValue::make(); + case T_BYTE: return ByteFieldValue::make(); case T_PREDICATE: return std::make_unique<PredicateFieldValue>(); } LOG_ABORT("getId() returned value out of range"); diff --git a/document/src/vespa/document/fieldvalue/arrayfieldvalue.h b/document/src/vespa/document/fieldvalue/arrayfieldvalue.h index b37049f207a..c4f879f7348 100644 --- a/document/src/vespa/document/fieldvalue/arrayfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/arrayfieldvalue.h @@ -74,11 +74,11 @@ public: // Iterator functionality const_iterator begin() const { return array().begin(); } const_iterator end() const { return array().end(); } - iterator begin() { return array().begin(); } - iterator end() { return array().end(); } DECLARE_IDENTIFIABLE_ABSTRACT(ArrayFieldValue); private: + iterator begin() { return array().begin(); } + iterator end() { return array().end(); } const IArray & array() const { return *_array; } IArray & array() { return *_array; } }; diff --git a/document/src/vespa/document/fieldvalue/boolfieldvalue.cpp b/document/src/vespa/document/fieldvalue/boolfieldvalue.cpp index c542936825d..1c5261877fd 100644 --- a/document/src/vespa/document/fieldvalue/boolfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/boolfieldvalue.cpp @@ -87,25 +87,5 @@ BoolFieldValue::operator=(vespalib::stringref v) { _value = (v == "true"); return *this; } -BoolFieldValue& -BoolFieldValue::operator=(int32_t v) { - _value = (v != 0); - return *this; -} -BoolFieldValue& -BoolFieldValue::operator=(int64_t v) { - _value = (v != 0); - return *this; -} -BoolFieldValue& -BoolFieldValue::operator=(float v) { - _value = (v != 0); - return *this; -} -BoolFieldValue& -BoolFieldValue::operator=(double v) { - _value = (v != 0); - return *this; -} } // namespace document diff --git a/document/src/vespa/document/fieldvalue/boolfieldvalue.h b/document/src/vespa/document/fieldvalue/boolfieldvalue.h index d03ded5c9f1..01cfcf2cd48 100644 --- a/document/src/vespa/document/fieldvalue/boolfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/boolfieldvalue.h @@ -42,12 +42,8 @@ public: vespalib::string getAsString() const override; BoolFieldValue& operator=(vespalib::stringref) override; - BoolFieldValue& operator=(int32_t) override; - BoolFieldValue& operator=(int64_t) override; - BoolFieldValue& operator=(float) override; - BoolFieldValue& operator=(double) override; - DECLARE_IDENTIFIABLE(BoolFieldValue); + static std::unique_ptr<BoolFieldValue> make(bool value=false) { return std::make_unique<BoolFieldValue>(value); } }; } diff --git a/document/src/vespa/document/fieldvalue/bytefieldvalue.h b/document/src/vespa/document/fieldvalue/bytefieldvalue.h index 7f6bc429b69..2cccf483c84 100644 --- a/document/src/vespa/document/fieldvalue/bytefieldvalue.h +++ b/document/src/vespa/document/fieldvalue/bytefieldvalue.h @@ -14,7 +14,6 @@ namespace document { class ByteFieldValue : public NumericFieldValue<int8_t> { public: - typedef std::unique_ptr<ByteFieldValue> UP; typedef int8_t Number; ByteFieldValue(Number value = 0) @@ -24,10 +23,10 @@ public: void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); } const DataType *getDataType() const override { return DataType::BYTE; } ByteFieldValue* clone() const override { return new ByteFieldValue(*this); } - using NumericFieldValue<Number>::operator=; + using NumericFieldValue<Number>::operator=; DECLARE_IDENTIFIABLE(ByteFieldValue); - + static std::unique_ptr<ByteFieldValue> make(Number value=0) { return std::make_unique<ByteFieldValue>(value); } }; } // document diff --git a/document/src/vespa/document/fieldvalue/collectionfieldvalue.h b/document/src/vespa/document/fieldvalue/collectionfieldvalue.h index fcdd7d72ebd..9efd3b91bc6 100644 --- a/document/src/vespa/document/fieldvalue/collectionfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/collectionfieldvalue.h @@ -62,41 +62,6 @@ public: virtual size_t size() const = 0; virtual void clear() = 0; - // Convenience functions for using primitives directly - - bool add(vespalib::stringref val) - { return addValue(*createNested() = val); } - bool add(int32_t val) - { return addValue(*createNested() = val); } - bool add(int64_t val) - { return addValue(*createNested() = val); } - bool add(float val) - { return addValue(*createNested() = val); } - bool add(double val) - { return addValue(*createNested() = val); } - - bool contains(vespalib::stringref val) - { return containsValue(*createNested() = val); } - bool contains(int32_t val) - { return containsValue(*createNested() = val); } - bool contains(int64_t val) - { return containsValue(*createNested() = val); } - bool contains(float val) - { return containsValue(*createNested() = val); } - bool contains(double val) - { return containsValue(*createNested() = val); } - - bool remove(vespalib::stringref val) - { return removeValue(*createNested() = val); } - bool remove(int32_t val) - { return removeValue(*createNested() = val); } - bool remove(int64_t val) - { return removeValue(*createNested() = val); } - bool remove(float val) - { return removeValue(*createNested() = val); } - bool remove(double val) - { return removeValue(*createNested() = val); } - DECLARE_IDENTIFIABLE_ABSTRACT(CollectionFieldValue); }; diff --git a/document/src/vespa/document/fieldvalue/doublefieldvalue.h b/document/src/vespa/document/fieldvalue/doublefieldvalue.h index feb15e52223..b9b20bbd24a 100644 --- a/document/src/vespa/document/fieldvalue/doublefieldvalue.h +++ b/document/src/vespa/document/fieldvalue/doublefieldvalue.h @@ -14,7 +14,6 @@ namespace document { class DoubleFieldValue : public NumericFieldValue<double> { public: - typedef std::unique_ptr<DoubleFieldValue> UP; typedef double Number; DoubleFieldValue(Number value = 0) : NumericFieldValue<Number>(value) {} @@ -26,9 +25,8 @@ public: DoubleFieldValue* clone() const override { return new DoubleFieldValue(*this); } using NumericFieldValue<Number>::operator=; - DECLARE_IDENTIFIABLE(DoubleFieldValue); - + static std::unique_ptr<DoubleFieldValue> make(Number value=0) { return std::make_unique<DoubleFieldValue>(value); } }; } // document diff --git a/document/src/vespa/document/fieldvalue/fieldvalue.cpp b/document/src/vespa/document/fieldvalue/fieldvalue.cpp index 8a678ddf968..c69b7169aa0 100644 --- a/document/src/vespa/document/fieldvalue/fieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/fieldvalue.cpp @@ -95,30 +95,6 @@ FieldValue::operator=(vespalib::stringref) throw IllegalArgumentException("Cannot assign string to datatype " + getDataType()->toString(), VESPA_STRLOC); } -FieldValue& -FieldValue::operator=(int32_t) -{ - throw IllegalArgumentException("Cannot assign int to datatype " + getDataType()->toString(), VESPA_STRLOC); -} - -FieldValue& -FieldValue::operator=(int64_t) -{ - throw IllegalArgumentException("Cannot assign long to datatype " + getDataType()->toString(), VESPA_STRLOC); -} - -FieldValue& -FieldValue::operator=(float) -{ - throw IllegalArgumentException("Cannot assign float to datatype " + getDataType()->toString(), VESPA_STRLOC); -} - -FieldValue& -FieldValue::operator=(double) -{ - throw IllegalArgumentException("Cannot assign double to datatype " + getDataType()->toString(), VESPA_STRLOC); -} - char FieldValue::getAsByte() const { diff --git a/document/src/vespa/document/fieldvalue/fieldvalue.h b/document/src/vespa/document/fieldvalue/fieldvalue.h index b0507a6c251..fedb0141391 100644 --- a/document/src/vespa/document/fieldvalue/fieldvalue.h +++ b/document/src/vespa/document/fieldvalue/fieldvalue.h @@ -24,7 +24,6 @@ namespace document { namespace fieldvalue { class IteratorHandler; } -class ByteBuffer; class DataType; class FieldValue : public vespalib::Identifiable @@ -108,13 +107,6 @@ public: /** Override toXml from XmlSerializable to add start/stop tags. */ virtual std::string toXml(const std::string& indent = "") const; - // Utility functions to set commonly used value types. - virtual FieldValue& operator=(vespalib::stringref); - virtual FieldValue& operator=(int32_t); - virtual FieldValue& operator=(int64_t); - virtual FieldValue& operator=(float); - virtual FieldValue& operator=(double); - // Utility functions to unwrap field values if you know the type. /** @@ -187,6 +179,8 @@ public: std::string toString(bool verbose=false, const std::string& indent="") const; virtual void printXml(XmlOutputStream& out) const = 0; + // Utility functions to set commonly used value types. + virtual FieldValue& operator=(vespalib::stringref); private: fieldvalue::ModificationStatus iterateNested(FieldPath::const_iterator start, FieldPath::const_iterator end, fieldvalue::IteratorHandler & handler) const { diff --git a/document/src/vespa/document/fieldvalue/floatfieldvalue.h b/document/src/vespa/document/fieldvalue/floatfieldvalue.h index f33939d8d67..f662c400633 100644 --- a/document/src/vespa/document/fieldvalue/floatfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/floatfieldvalue.h @@ -14,7 +14,6 @@ namespace document { class FloatFieldValue : public NumericFieldValue<float> { public: - typedef std::unique_ptr<FloatFieldValue> UP; typedef float Number; FloatFieldValue(Number value = 0) : NumericFieldValue<Number>(value) {} @@ -26,9 +25,8 @@ public: FloatFieldValue* clone() const override { return new FloatFieldValue(*this); } using NumericFieldValue<Number>::operator=; - DECLARE_IDENTIFIABLE(FloatFieldValue); - + static std::unique_ptr<FloatFieldValue> make(Number value = 0) { return std::make_unique<FloatFieldValue>(value); } }; } // document diff --git a/document/src/vespa/document/fieldvalue/intfieldvalue.h b/document/src/vespa/document/fieldvalue/intfieldvalue.h index dbe419379a1..d365e34779e 100644 --- a/document/src/vespa/document/fieldvalue/intfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/intfieldvalue.h @@ -14,7 +14,6 @@ namespace document { class IntFieldValue : public NumericFieldValue<int32_t> { public: - typedef std::unique_ptr<IntFieldValue> UP; typedef int32_t Number; IntFieldValue(Number value = 0) : NumericFieldValue<Number>(value) {} @@ -27,7 +26,7 @@ public: using NumericFieldValue<Number>::operator=; DECLARE_IDENTIFIABLE(IntFieldValue); - + static std::unique_ptr<IntFieldValue> make(Number value=0) { return std::make_unique<IntFieldValue>(value); } }; } // document diff --git a/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp b/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp index 9b648400533..7ba537d9cc2 100644 --- a/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp @@ -1,10 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "literalfieldvalue.h" #include "literalfieldvalue.hpp" #include <vespa/document/util/stringutil.h> #include <vespa/vespalib/util/xmlstream.h> -#include <sstream> using namespace vespalib::xml; @@ -21,7 +19,7 @@ LiteralFieldValueB::LiteralFieldValueB() : _value = _backing; } -LiteralFieldValueB::~LiteralFieldValueB() { } +LiteralFieldValueB::~LiteralFieldValueB() = default; LiteralFieldValueB::LiteralFieldValueB(const LiteralFieldValueB& other) : FieldValue(other), @@ -32,7 +30,7 @@ LiteralFieldValueB::LiteralFieldValueB(const LiteralFieldValueB& other) _value = _backing; } -LiteralFieldValueB::LiteralFieldValueB(const string& value) +LiteralFieldValueB::LiteralFieldValueB(const stringref & value) : FieldValue(), _value(), _backing(value), @@ -117,44 +115,6 @@ LiteralFieldValueB::syncBacking() const _value = _backing; } - -namespace { -template <typename T> -std::string valueToString(T value) { - std::ostringstream ost; - ost << value; - return ost.str(); -} -} // namespace - -FieldValue& -LiteralFieldValueB::operator=(int32_t value) -{ - setValue(valueToString(value)); - return *this; -} - -FieldValue& -LiteralFieldValueB::operator=(int64_t value) -{ - setValue(valueToString(value)); - return *this; -} - -FieldValue& -LiteralFieldValueB::operator=(float value) -{ - setValue(valueToString(value)); - return *this; -} - -FieldValue& -LiteralFieldValueB::operator=(double value) -{ - setValue(valueToString(value)); - return *this; -} - template class LiteralFieldValue<RawFieldValue, DataType::T_RAW, false>; template class LiteralFieldValue<StringFieldValue, DataType::T_STRING, true>; diff --git a/document/src/vespa/document/fieldvalue/literalfieldvalue.h b/document/src/vespa/document/fieldvalue/literalfieldvalue.h index e84f0c529c7..6e3f0223b20 100644 --- a/document/src/vespa/document/fieldvalue/literalfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/literalfieldvalue.h @@ -32,7 +32,7 @@ public: ~LiteralFieldValueB(); LiteralFieldValueB(const LiteralFieldValueB &); - LiteralFieldValueB(const string& value); + LiteralFieldValueB(const stringref & value); const value_type & getValue() const { sync(); return _backing; } /** @@ -69,10 +69,6 @@ public: bool hasChanged() const override{ return _altered; } FieldValue& operator=(vespalib::stringref) override; - FieldValue& operator=(int32_t) override; - FieldValue& operator=(int64_t) override; - FieldValue& operator=(float) override; - FieldValue& operator=(double) override; protected: void syncBacking() const __attribute__((noinline)); void sync() const { @@ -95,7 +91,7 @@ public: typedef std::unique_ptr<SubClass> UP; LiteralFieldValue() : LiteralFieldValueB() { } - LiteralFieldValue(const string& value) : LiteralFieldValueB(value) { } + LiteralFieldValue(const stringref& value) : LiteralFieldValueB(value) { } const DataType *getDataType() const override; }; diff --git a/document/src/vespa/document/fieldvalue/longfieldvalue.h b/document/src/vespa/document/fieldvalue/longfieldvalue.h index 12a0615e0ad..64ab4332cef 100644 --- a/document/src/vespa/document/fieldvalue/longfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/longfieldvalue.h @@ -14,7 +14,6 @@ namespace document { class LongFieldValue : public NumericFieldValue<int64_t> { public: - typedef std::unique_ptr<LongFieldValue> UP; typedef int64_t Number; LongFieldValue(Number value = 0) : NumericFieldValue<Number>(value) {} @@ -26,8 +25,8 @@ public: LongFieldValue* clone() const override { return new LongFieldValue(*this); } using NumericFieldValue<Number>::operator=; - DECLARE_IDENTIFIABLE(LongFieldValue); + static std::unique_ptr<LongFieldValue> make(Number value=0) { return std::make_unique<LongFieldValue>(value); } }; diff --git a/document/src/vespa/document/fieldvalue/numericfieldvalue.h b/document/src/vespa/document/fieldvalue/numericfieldvalue.h index c094cf0689c..0a557af93d9 100644 --- a/document/src/vespa/document/fieldvalue/numericfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/numericfieldvalue.h @@ -41,10 +41,6 @@ public: int fastCompare(const FieldValue& other) const override final; FieldValue& operator=(vespalib::stringref) override; - FieldValue& operator=(int32_t) override; - FieldValue& operator=(int64_t) override; - FieldValue& operator=(float) override; - FieldValue& operator=(double) override; size_t hash() const override final { return vespalib::hash<Number>()(_value); } char getAsByte() const override; diff --git a/document/src/vespa/document/fieldvalue/numericfieldvalue.hpp b/document/src/vespa/document/fieldvalue/numericfieldvalue.hpp index c4f1fdf13a1..f1bbce5450d 100644 --- a/document/src/vespa/document/fieldvalue/numericfieldvalue.hpp +++ b/document/src/vespa/document/fieldvalue/numericfieldvalue.hpp @@ -123,42 +123,6 @@ NumericFieldValue<Number>::operator=(vespalib::stringref value) } template<typename Number> -FieldValue& -NumericFieldValue<Number>::operator=(int32_t value) -{ - _value = static_cast<Number>(value); - _altered = true; - return *this; -} - -template<typename Number> -FieldValue& -NumericFieldValue<Number>::operator=(int64_t value) -{ - _value = static_cast<Number>(value); - _altered = true; - return *this; -} - -template<typename Number> -FieldValue& -NumericFieldValue<Number>::operator=(float value) -{ - _value = static_cast<Number>(value); - _altered = true; - return *this; -} - -template<typename Number> -FieldValue& -NumericFieldValue<Number>::operator=(double value) -{ - _value = static_cast<Number>(value); - _altered = true; - return *this; -} - -template<typename Number> char NumericFieldValue<Number>::getAsByte() const { diff --git a/document/src/vespa/document/fieldvalue/shortfieldvalue.h b/document/src/vespa/document/fieldvalue/shortfieldvalue.h index f2047d1a521..fa61c37c4eb 100644 --- a/document/src/vespa/document/fieldvalue/shortfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/shortfieldvalue.h @@ -27,9 +27,8 @@ public: ShortFieldValue* clone() const override { return new ShortFieldValue(*this); } using NumericFieldValue<Number>::operator=; - DECLARE_IDENTIFIABLE(ShortFieldValue); - + static std::unique_ptr<ShortFieldValue> make(int16_t value = 0) { return std::make_unique<ShortFieldValue>(value); } }; } // document diff --git a/document/src/vespa/document/fieldvalue/stringfieldvalue.h b/document/src/vespa/document/fieldvalue/stringfieldvalue.h index 15aeeccd0de..17a0302f6f9 100644 --- a/document/src/vespa/document/fieldvalue/stringfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/stringfieldvalue.h @@ -24,7 +24,7 @@ public: typedef std::vector<SpanTree::UP> SpanTrees; StringFieldValue() : Parent(), _annotationData() { } - StringFieldValue(const string &value) + StringFieldValue(const vespalib::stringref &value) : Parent(value), _annotationData() { } StringFieldValue(const StringFieldValue &rhs); @@ -57,6 +57,8 @@ public: using LiteralFieldValueB::operator=; DECLARE_IDENTIFIABLE(StringFieldValue); + static std::unique_ptr<StringFieldValue> make(vespalib::stringref value) { return std::make_unique<StringFieldValue>(value); } + static std::unique_ptr<StringFieldValue> make() { return StringFieldValue::make(""); } private: void doClearSpanTrees(); diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp index 555964d8b34..fbe6dadb320 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp @@ -79,14 +79,16 @@ bool StructFieldValue::serializeField(int field_id, uint16_t version, FieldValue } } -void StructFieldValue::getRawFieldIds(vector<int> &raw_ids) const { - raw_ids.clear(); +vector<int> +StructFieldValue::getRawFieldIds() const { + vector<int> raw_ids; raw_ids.reserve(_fields.getEntries().size()); for (const SerializableArray::Entry & entry : _fields.getEntries()) { raw_ids.emplace_back(entry.id()); } sort(raw_ids.begin(), raw_ids.end()); raw_ids.erase(unique(raw_ids.begin(), raw_ids.end()), raw_ids.end()); + return raw_ids; } void @@ -242,10 +244,8 @@ StructFieldValue::compare(const FieldValue& otherOrg) const } const auto & other = static_cast<const StructFieldValue&>(otherOrg); - std::vector<int> a; - getRawFieldIds(a); - std::vector<int> b; - other.getRawFieldIds(b); + std::vector<int> a = getRawFieldIds(); + std::vector<int> b = other.getRawFieldIds(); for (size_t i(0); i < std::min(a.size(), b.size()); i++) { if (a[i] != b[i]) { @@ -337,12 +337,9 @@ struct StructFieldValue::FieldIterator : public StructuredIterator { explicit FieldIterator(const StructFieldValue& s) : _struct(s), - _ids(), + _ids(s.getRawFieldIds()), _cur(_ids.begin()) - { - s.getRawFieldIds(_ids); - _cur = _ids.begin(); - } + { } void skipTo(int fieldId) { while (_cur != _ids.end() && fieldId != *_cur) { diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.h b/document/src/vespa/document/fieldvalue/structfieldvalue.h index ab35dc04421..24e143ddc27 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.h @@ -54,7 +54,7 @@ public: uint16_t getVersion() const { return _version; } // raw_ids may contain ids for elements not in the struct's datatype. - void getRawFieldIds(std::vector<int> &raw_ids) const; + std::vector<int> getRawFieldIds() const; void getRawFieldIds(std::vector<int> &raw_ids, const FieldSet& fieldSet) const; void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); } @@ -98,7 +98,6 @@ private: VESPA_DLL_LOCAL const StructDataType & getStructType() const; struct FieldIterator; - friend struct FieldIterator; StructuredIterator::UP getIterator(const Field* toFind) const override; diff --git a/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp index becdfdabb5f..53f75cb2e73 100644 --- a/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp @@ -172,20 +172,6 @@ StructuredFieldValue::onIterateNested(PathRange nested, IteratorHandler & handle } } -using ConstCharP = const char *; -template void StructuredFieldValue::set(const Field& field, int32_t value); -template void StructuredFieldValue::set(const Field& field, int64_t value); -template void StructuredFieldValue::set(const Field& field, double value); -template void StructuredFieldValue::set(const Field& field, ConstCharP value); -template void StructuredFieldValue::set(const Field& field, vespalib::stringref value); -template void StructuredFieldValue::set(const Field& field, vespalib::string value); -template void StructuredFieldValue::set(vespalib::stringref field, int32_t value); -template void StructuredFieldValue::set(vespalib::stringref field, int64_t value); -template void StructuredFieldValue::set(vespalib::stringref field, double value); -template void StructuredFieldValue::set(vespalib::stringref field, ConstCharP value); -template void StructuredFieldValue::set(vespalib::stringref field, vespalib::stringref value); -template void StructuredFieldValue::set(vespalib::stringref field, vespalib::string value); - template std::unique_ptr<MapFieldValue> StructuredFieldValue::getAs<MapFieldValue>(const Field &field) const; template std::unique_ptr<ArrayFieldValue> StructuredFieldValue::getAs<ArrayFieldValue>(const Field &field) const; template std::unique_ptr<WeightedSetFieldValue> StructuredFieldValue::getAs<WeightedSetFieldValue>(const Field &field) const; diff --git a/document/src/vespa/document/fieldvalue/structuredfieldvalue.h b/document/src/vespa/document/fieldvalue/structuredfieldvalue.h index b1b88323a3b..9d79b6279a4 100644 --- a/document/src/vespa/document/fieldvalue/structuredfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/structuredfieldvalue.h @@ -152,6 +152,12 @@ public: void setValue(const Field& field, FieldValue::UP value) { setFieldValue(field, std::move(value)); } + void setValue(vespalib::stringref fieldName, const FieldValue& value) { + setFieldValue(getField(fieldName), value); + } + void setValue(vespalib::stringref fieldName, FieldValue::UP value) { + setFieldValue(getField(fieldName), std::move(value)); + } /** Remove the value of given field if it is set. */ //These are affected by the begin/commitTanasaction @@ -166,13 +172,6 @@ public: void remove(vespalib::stringref fieldName) { removeFieldValue(getField(fieldName)); } - void setValue(vespalib::stringref fieldName, const FieldValue& value) { - setFieldValue(getField(fieldName), value); - } - template<typename PrimitiveType> - void set(const Field& field, PrimitiveType value); - template<typename PrimitiveType> - void set(vespalib::stringref fieldName, PrimitiveType value); size_t getSetFieldCount() const { size_t count = 0; diff --git a/document/src/vespa/document/fieldvalue/structuredfieldvalue.hpp b/document/src/vespa/document/fieldvalue/structuredfieldvalue.hpp index df02506c076..4b347d7cc07 100644 --- a/document/src/vespa/document/fieldvalue/structuredfieldvalue.hpp +++ b/document/src/vespa/document/fieldvalue/structuredfieldvalue.hpp @@ -20,20 +20,4 @@ StructuredFieldValue::getAs(const Field &field) const { return std::unique_ptr<T>(t); } -template<typename PrimitiveType> -void -StructuredFieldValue::set(const Field& field, PrimitiveType value) -{ - FieldValue::UP fval(field.getDataType().createFieldValue()); - *fval = value; - setFieldValue(field, std::move(fval)); -} - -template<typename PrimitiveType> -void -StructuredFieldValue::set(vespalib::stringref fieldName, PrimitiveType value) -{ - set(getField(fieldName), value); -} - } // document diff --git a/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.cpp b/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.cpp index 6b80ceb1f6e..2a3726095df 100644 --- a/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.cpp @@ -59,7 +59,7 @@ WeightedSetFieldValue::add(const FieldValue& key, int weight) _map.erase(key); return false; } - return _map.insert(FieldValue::UP(key.clone()), std::make_unique<IntFieldValue>(weight)); + return _map.insert(FieldValue::UP(key.clone()), IntFieldValue::make(weight)); } bool @@ -67,14 +67,14 @@ WeightedSetFieldValue::addIgnoreZeroWeight(const FieldValue& key, int32_t weight { verifyKey(key); _altered = true; - return _map.insert(FieldValue::UP(key.clone()), std::make_unique<IntFieldValue>(weight)); + return _map.insert(FieldValue::UP(key.clone()), IntFieldValue::make(weight)); } void WeightedSetFieldValue::push_back(FieldValue::UP key, int weight) { _altered = true; - _map.push_back(std::move(key), std::make_unique<IntFieldValue>(weight)); + _map.push_back(std::move(key), IntFieldValue::make(weight)); } void diff --git a/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.h b/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.h index b88c44085ba..d58819607b4 100644 --- a/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.h @@ -64,8 +64,7 @@ public: bool addIgnoreZeroWeight(const FieldValue&, int32_t weight = 1); void push_back(FieldValue::UP, int32_t weight); void increment(const FieldValue& fval, int val = 1); - void decrement(const FieldValue& fval, int val = 1) - { increment(fval, -1*val); } + void decrement(const FieldValue& fval, int val = 1) { increment(fval, -1*val); } int32_t get(const FieldValue&, int32_t defaultValue = 0) const; bool isEmpty() const override { return _map.isEmpty(); } @@ -76,13 +75,12 @@ public: FieldValue& assign(const FieldValue&) override; WeightedSetFieldValue* clone() const override { return new WeightedSetFieldValue(*this); } - virtual int compare(const FieldValue&) const override; - virtual void printXml(XmlOutputStream& out) const override; - virtual void print(std::ostream& out, bool verbose, - const std::string& indent) const override; - virtual bool hasChanged() const override; + int compare(const FieldValue&) const override; + void printXml(XmlOutputStream& out) const override; + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + bool hasChanged() const override; - // Implements iterating through internal content. + // Implements iterating through internal content. typedef WeightedFieldValueMap::const_iterator const_iterator; typedef WeightedFieldValueMap::iterator iterator; @@ -95,54 +93,7 @@ public: const_iterator find(const FieldValue& fv) const; iterator find(const FieldValue& fv); - // Utility functions for easy use of weighted sets of primitives - - bool add(vespalib::stringref val, int32_t weight = 1) - { return add(*createNested() = val, weight); } - bool add(int32_t val, int32_t weight = 1) - { return add(*createNested() = val, weight); } - bool add(int64_t val, int32_t weight = 1) - { return add(*createNested() = val, weight); } - bool add(float val, int32_t weight = 1) - { return add(*createNested() = val, weight); } - bool add(double val, int32_t weight = 1) - { return add(*createNested() = val, weight); } - - int32_t get(vespalib::stringref val) const - { return get(*createNested() = val); } - int32_t get(int32_t val) const - { return get(*createNested() = val); } - int32_t get(int64_t val) const - { return get(*createNested() = val); } - int32_t get(float val) const - { return get(*createNested() = val); } - int32_t get(double val) const - { return get(*createNested() = val); } - - void increment(vespalib::stringref val, int32_t weight = 1) - { increment(*createNested() = val, weight); } - void increment(int32_t val, int32_t weight = 1) - { increment(*createNested() = val, weight); } - void increment(int64_t val, int32_t weight = 1) - { increment(*createNested() = val, weight); } - void increment(float val, int32_t weight = 1) - { increment(*createNested() = val, weight); } - void increment(double val, int32_t weight = 1) - { increment(*createNested() = val, weight); } - - void decrement(vespalib::stringref val, int32_t weight = 1) - { decrement(*createNested() = val, weight); } - void decrement(int32_t val, int32_t weight = 1) - { decrement(*createNested() = val, weight); } - void decrement(int64_t val, int32_t weight = 1) - { decrement(*createNested() = val, weight); } - void decrement(float val, int32_t weight = 1) - { decrement(*createNested() = val, weight); } - void decrement(double val, int32_t weight = 1) - { decrement(*createNested() = val, weight); } - DECLARE_IDENTIFIABLE_ABSTRACT(WeightedSetFieldValue); - }; } // document diff --git a/document/src/vespa/document/test/fieldvalue_helpers.h b/document/src/vespa/document/test/fieldvalue_helpers.h new file mode 100644 index 00000000000..a7d899b385f --- /dev/null +++ b/document/src/vespa/document/test/fieldvalue_helpers.h @@ -0,0 +1,71 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/document/fieldvalue/collectionfieldvalue.h> +#include <vespa/document/fieldvalue/weightedsetfieldvalue.h> +#include <vespa/document/fieldvalue/longfieldvalue.h> +#include <vespa/document/fieldvalue/floatfieldvalue.h> +#include <vespa/document/fieldvalue/doublefieldvalue.h> + +namespace document { + +class CollectionHelper { +public: + CollectionHelper(CollectionFieldValue &value) : _cfv(value) {} + + // Convenience functions for using primitives directly + bool add(vespalib::stringref val) { return _cfv.add(*_cfv.createNested() = val); } + bool add(int32_t val) { return _cfv.add(_cfv.createNested()->assign(IntFieldValue(val))); } + bool add(int64_t val) { return _cfv.add(_cfv.createNested()->assign(LongFieldValue(val))); } + bool add(float val) { return _cfv.add(_cfv.createNested()->assign(FloatFieldValue(val))); } + bool add(double val) { return _cfv.add(_cfv.createNested()->assign(DoubleFieldValue(val))); } + + bool contains(vespalib::stringref val) { return _cfv.contains(*_cfv.createNested() = val); } + bool contains(int32_t val) { return _cfv.contains(_cfv.createNested()->assign(IntFieldValue(val))); } + bool contains(int64_t val) { return _cfv.contains(_cfv.createNested()->assign(LongFieldValue(val))); } + bool contains(float val) { return _cfv.contains(_cfv.createNested()->assign(FloatFieldValue(val))); } + bool contains(double val) { return _cfv.contains(_cfv.createNested()->assign(DoubleFieldValue(val))); } + + bool remove(vespalib::stringref val) { return _cfv.remove(*_cfv.createNested() = val); } + bool remove(int32_t val) { return _cfv.remove(_cfv.createNested()->assign(IntFieldValue(val))); } + bool remove(int64_t val) { return _cfv.remove(_cfv.createNested()->assign(LongFieldValue(val))); } + bool remove(float val) { return _cfv.remove(_cfv.createNested()->assign(FloatFieldValue(val))); } + bool remove(double val) { return _cfv.remove(_cfv.createNested()->assign(DoubleFieldValue(val))); } +private: + CollectionFieldValue & _cfv; +}; + +class WSetHelper { +public: + WSetHelper(WeightedSetFieldValue & ws) : _ws(ws) { } + + // Utility functions for easy use of weighted sets of primitives + + bool add(vespalib::stringref val, int32_t weight = 1) { return _ws.add(*_ws.createNested() = val, weight); } + bool add(int32_t val, int32_t weight = 1) { return _ws.add(_ws.createNested()->assign(IntFieldValue(val)), weight); } + bool add(int64_t val, int32_t weight = 1) { return _ws.add(_ws.createNested()->assign(LongFieldValue(val)), weight); } + bool add(float val, int32_t weight = 1) { return _ws.add(_ws.createNested()->assign(FloatFieldValue(val)), weight); } + bool add(double val, int32_t weight = 1) { return _ws.add(_ws.createNested()->assign(DoubleFieldValue(val)), weight); } + + int32_t get(vespalib::stringref val) const { return _ws.get(*_ws.createNested() = val); } + int32_t get(int32_t val) const { return _ws.get(_ws.createNested()->assign(IntFieldValue(val))); } + int32_t get(int64_t val) const { return _ws.get(_ws.createNested()->assign(LongFieldValue(val))); } + int32_t get(float val) const { return _ws.get(_ws.createNested()->assign(FloatFieldValue(val))); } + int32_t get(double val) const { return _ws.get(_ws.createNested()->assign(DoubleFieldValue(val))); } + + void increment(vespalib::stringref val, int32_t weight = 1) { _ws.increment(*_ws.createNested() = val, weight); } + void increment(int32_t val, int32_t weight = 1) { _ws.increment(_ws.createNested()->assign(IntFieldValue(val)), weight); } + void increment(int64_t val, int32_t weight = 1) { _ws.increment(_ws.createNested()->assign(LongFieldValue(val)), weight); } + void increment(float val, int32_t weight = 1) { _ws.increment(_ws.createNested()->assign(FloatFieldValue(val)), weight); } + void increment(double val, int32_t weight = 1) { _ws.increment(_ws.createNested()->assign(DoubleFieldValue(val)), weight); } + + void decrement(vespalib::stringref val, int32_t weight = 1) { _ws.decrement(*_ws.createNested() = val, weight); } + void decrement(int32_t val, int32_t weight = 1) { _ws.decrement(_ws.createNested()->assign(IntFieldValue(val)), weight); } + void decrement(int64_t val, int32_t weight = 1) { _ws.decrement(_ws.createNested()->assign(LongFieldValue(val)), weight); } + void decrement(float val, int32_t weight = 1) { _ws.decrement(_ws.createNested()->assign(FloatFieldValue(val)), weight); } + void decrement(double val, int32_t weight = 1) { _ws.decrement(_ws.createNested()->assign(DoubleFieldValue(val)), weight); } +private: + WeightedSetFieldValue & _ws; +}; +} diff --git a/documentapi/src/tests/messages/messages60test.cpp b/documentapi/src/tests/messages/messages60test.cpp index c0bdb71fb25..a7604888c69 100644 --- a/documentapi/src/tests/messages/messages60test.cpp +++ b/documentapi/src/tests/messages/messages60test.cpp @@ -10,6 +10,7 @@ #include <vespa/document/update/fieldpathupdates.h> #include <vespa/documentapi/documentapi.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> +#include <vespa/document/fieldvalue/document.h> using document::DataType; using document::DocumentTypeRepo; diff --git a/documentapi/src/tests/policies/policies_test.cpp b/documentapi/src/tests/policies/policies_test.cpp index 61716e4ed6b..dfa9cbb581e 100644 --- a/documentapi/src/tests/policies/policies_test.cpp +++ b/documentapi/src/tests/policies/policies_test.cpp @@ -18,11 +18,11 @@ #include <vespa/messagebus/testlib/testserver.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/document/base/testdocrepo.h> -#include <vespa/document/config/documenttypes_config_fwd.h> #include <vespa/document/fieldvalue/longfieldvalue.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/update/documentupdate.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/stringfmt.h> #include <thread> diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp index 534a4c62efc..1717b687322 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/util/growablebytebuffer.h> #include <vespa/document/util/bytebuffer.h> #include <vespa/document/fieldset/fieldsets.h> +#include <vespa/document/fieldvalue/document.h> #include <climits> using document::FixedBucketSpaces; diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h index 48fece57064..0b43b37c0b7 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h @@ -9,9 +9,8 @@ #include <vespa/vdslib/container/visitorstatistics.h> #include <vespa/document/bucket/bucketid.h> #include <vespa/documentapi/messagebus/documentprotocol.h> -#include <vespa/document/fieldvalue/document.h> - +namespace document { class Document; } namespace documentapi { typedef uint64_t Timestamp; @@ -237,25 +236,25 @@ public: */ class DocumentListMessage : public VisitorMessage { public: - typedef std::unique_ptr<DocumentListMessage> UP; + using UP = std::unique_ptr<DocumentListMessage>; + using DocumentSP = std::shared_ptr<document::Document>; class Entry { public: Entry(); - Entry(int64_t timestamp, document::Document::SP doc, bool removeEntry); + Entry(int64_t timestamp, DocumentSP doc, bool removeEntry); Entry(const Entry& other); Entry(const document::DocumentTypeRepo &repo, document::ByteBuffer& buf); int64_t getTimestamp() { return _timestamp; } - const document::Document::SP& getDocument() { return _document; } + const DocumentSP & getDocument() { return _document; } bool isRemoveEntry() { return _removeEntry; } void serialize(vespalib::GrowableByteBuffer& buf) const; - uint32_t getSerializedSize() const; private: - int64_t _timestamp; - document::Document::SP _document; - bool _removeEntry; + int64_t _timestamp; + DocumentSP _document; + bool _removeEntry; }; private: diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.cpp index 5fded6a6266..ae4041732a7 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.cpp @@ -3,6 +3,7 @@ #include "contentpolicy.h" #include <vespa/document/base/documentid.h> #include <vespa/document/update/documentupdate.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/messagebus/emptyreply.h> #include <vespa/messagebus/error.h> #include <vespa/documentapi/documentapi.h> diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp index 8b7c56117b1..ad32a5b4fa7 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp @@ -6,6 +6,7 @@ #include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/document/select/parser.h> #include <vespa/document/update/documentupdate.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/documentapi/documentapi.h> #include <vespa/vespalib/objects/nbostream.h> diff --git a/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp b/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp index 868e9d036f1..a3aa7cbb32f 100644 --- a/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp +++ b/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp @@ -299,10 +299,10 @@ int main(int argc, char **argv) { try { return my_main(argc, argv); } catch (const MyError &err) { - fprintf(stderr, "error: %s\n", err.msg.c_str()); + fprintf(stdout, "error: %s\n", err.msg.c_str()); return 3; } catch (const std::exception &ex) { - fprintf(stderr, "got exception: %s\n", ex.what()); + fprintf(stdout, "got exception: %s\n", ex.what()); return 2; } } diff --git a/eval/src/tests/apps/analyze_onnx_model/analyze_onnx_model_test.cpp b/eval/src/tests/apps/analyze_onnx_model/analyze_onnx_model_test.cpp index 72ef0346ea3..68d9ed69f80 100644 --- a/eval/src/tests/apps/analyze_onnx_model/analyze_onnx_model_test.cpp +++ b/eval/src/tests/apps/analyze_onnx_model/analyze_onnx_model_test.cpp @@ -39,13 +39,13 @@ TEST_F("require that output types can be probed", ServerCmd(probe_cmd)) { //----------------------------------------------------------------------------- -TEST_F("test error: invalid json", ServerCmd(probe_cmd, ServerCmd::capture_stderr_tag())) { +TEST_F("test error: invalid json", ServerCmd(probe_cmd)) { auto out = f1.write_then_read_all("this is not valid json...\n"); EXPECT_TRUE(out.find("invalid json") < out.size()); EXPECT_EQUAL(f1.shutdown(), 3); } -TEST_F("test error: missing input type", ServerCmd(probe_cmd, ServerCmd::capture_stderr_tag())) { +TEST_F("test error: missing input type", ServerCmd(probe_cmd)) { Slime params; params.setObject(); params.get().setString("model", simple_model); @@ -55,7 +55,7 @@ TEST_F("test error: missing input type", ServerCmd(probe_cmd, ServerCmd::capture EXPECT_EQUAL(f1.shutdown(), 3); } -TEST_F("test error: invalid input type", ServerCmd(probe_cmd, ServerCmd::capture_stderr_tag())) { +TEST_F("test error: invalid input type", ServerCmd(probe_cmd)) { Slime params; params.setObject(); params.get().setString("model", simple_model); @@ -68,7 +68,7 @@ TEST_F("test error: invalid input type", ServerCmd(probe_cmd, ServerCmd::capture EXPECT_EQUAL(f1.shutdown(), 3); } -TEST_F("test error: incompatible input type", ServerCmd(probe_cmd, ServerCmd::capture_stderr_tag())) { +TEST_F("test error: incompatible input type", ServerCmd(probe_cmd)) { Slime params; params.setObject(); params.get().setString("model", simple_model); @@ -81,7 +81,7 @@ TEST_F("test error: incompatible input type", ServerCmd(probe_cmd, ServerCmd::ca EXPECT_EQUAL(f1.shutdown(), 3); } -TEST_F("test error: symbolic size mismatch", ServerCmd(probe_cmd, ServerCmd::capture_stderr_tag())) { +TEST_F("test error: symbolic size mismatch", ServerCmd(probe_cmd)) { Slime params; params.setObject(); params.get().setString("model", dynamic_model); @@ -97,4 +97,4 @@ TEST_F("test error: symbolic size mismatch", ServerCmd(probe_cmd, ServerCmd::cap //----------------------------------------------------------------------------- -TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/apps/eval_expr/eval_expr_test.cpp b/eval/src/tests/apps/eval_expr/eval_expr_test.cpp index fd103829618..d7979c6dbea 100644 --- a/eval/src/tests/apps/eval_expr/eval_expr_test.cpp +++ b/eval/src/tests/apps/eval_expr/eval_expr_test.cpp @@ -4,7 +4,6 @@ #include <vespa/vespalib/testkit/time_bomb.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/util/child_process.h> #include <vespa/vespalib/data/input.h> #include <vespa/vespalib/data/output.h> #include <vespa/vespalib/data/simple_buffer.h> @@ -168,4 +167,4 @@ TEST_F("require that type issues produces error", Server()) { //----------------------------------------------------------------------------- -TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/vespa/eval/eval/test/test_io.cpp b/eval/src/vespa/eval/eval/test/test_io.cpp index 044b6779431..5512ecf0c1c 100644 --- a/eval/src/vespa/eval/eval/test/test_io.cpp +++ b/eval/src/vespa/eval/eval/test/test_io.cpp @@ -66,59 +66,6 @@ StdOut::commit(size_t bytes) //----------------------------------------------------------------------------- -ChildIn::ChildIn(ChildProcess &child) - : _child(child), - _output() -{ -} - -WritableMemory -ChildIn::reserve(size_t bytes) -{ - return _output.reserve(bytes); -} - -Output & -ChildIn::commit(size_t bytes) -{ - _output.commit(bytes); - Memory buf = _output.obtain(); - REQUIRE(_child.write(buf.data, buf.size)); - _output.evict(buf.size); - return *this; -} - -//----------------------------------------------------------------------------- - -ChildOut::ChildOut(ChildProcess &child) - : _child(child), - _input() -{ - REQUIRE(_child.running()); - REQUIRE(!_child.failed()); -} - -Memory -ChildOut::obtain() -{ - if ((_input.get().size == 0) && !_child.eof()) { - WritableMemory buf = _input.reserve(4_Ki); - uint32_t res = _child.read(buf.data, buf.size); - REQUIRE((res > 0) || _child.eof()); - _input.commit(res); - } - return _input.obtain(); -} - -Input & -ChildOut::evict(size_t bytes) -{ - _input.evict(bytes); - return *this; -} - -//----------------------------------------------------------------------------- - void ServerCmd::maybe_close() { @@ -132,10 +79,8 @@ void ServerCmd::maybe_exit() { if (!_exited) { - read_until_eof(_child_stdout); - assert(_child.wait()); - assert(!_child.running()); - _exit_code = _child.getExitCode(); + read_until_eof(_child); + _exit_code = _child.join(); _exited = true; } } @@ -156,9 +101,7 @@ ServerCmd::dump_message(const char *prefix, const Slime &slime) } ServerCmd::ServerCmd(vespalib::string cmd) - : _child(cmd.c_str()), - _child_stdin(_child), - _child_stdout(_child), + : _child(cmd), _basename(fs::path(cmd).filename()), _closed(false), _exited(false), @@ -167,9 +110,7 @@ ServerCmd::ServerCmd(vespalib::string cmd) } ServerCmd::ServerCmd(vespalib::string cmd, capture_stderr_tag) - : _child(cmd.c_str(), ChildProcess::capture_stderr_tag()), - _child_stdin(_child), - _child_stdout(_child), + : _child(cmd, true), _basename(fs::path(cmd).filename()), _closed(false), _exited(false), @@ -187,9 +128,9 @@ Slime ServerCmd::invoke(const Slime &req) { dump_message("request --> ", req); - write_compact(req, _child_stdin); + write_compact(req, _child); Slime reply; - REQUIRE(JsonFormat::decode(_child_stdout, reply)); + REQUIRE(JsonFormat::decode(_child, reply)); dump_message("reply <-- ", reply); return reply; } @@ -199,12 +140,12 @@ ServerCmd::write_then_read_all(const vespalib::string &input) { vespalib::string result; dump_string("input --> ", input); - memcpy(_child_stdin.reserve(input.size()).data, input.data(), input.size()); - _child_stdin.commit(input.size()); + memcpy(_child.reserve(input.size()).data, input.data(), input.size()); + _child.commit(input.size()); maybe_close(); - for (auto mem = _child_stdout.obtain(); mem.size > 0; mem = _child_stdout.obtain()) { + for (auto mem = _child.obtain(); mem.size > 0; mem = _child.obtain()) { result.append(mem.data, mem.size); - _child_stdout.evict(mem.size); + _child.evict(mem.size); } dump_string("output <-- ", result); return result; diff --git a/eval/src/vespa/eval/eval/test/test_io.h b/eval/src/vespa/eval/eval/test/test_io.h index 62fd6588780..26fe260c857 100644 --- a/eval/src/vespa/eval/eval/test/test_io.h +++ b/eval/src/vespa/eval/eval/test/test_io.h @@ -9,7 +9,7 @@ #include <vespa/vespalib/data/simple_buffer.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/vespalib/util/child_process.h> +#include <vespa/vespalib/process/process.h> #include <functional> namespace vespalib::eval::test { @@ -40,38 +40,12 @@ public: }; /** - * Output adapter used to write to stdin of a child process. - **/ -class ChildIn : public Output { - ChildProcess &_child; - SimpleBuffer _output; -public: - ChildIn(ChildProcess &child); - WritableMemory reserve(size_t bytes) override; - Output &commit(size_t bytes) override; -}; - -/** - * Input adapter used to read from stdout of a child process. - **/ -class ChildOut : public Input { - ChildProcess &_child; - SimpleBuffer _input; -public: - ChildOut(ChildProcess &child); - Memory obtain() override; - Input &evict(size_t bytes) override; -}; - -/** * A command run as a child process that acts as a server reading json * from stdin and writing json to stdout. **/ class ServerCmd { private: - ChildProcess _child; - ChildIn _child_stdin; - ChildOut _child_stdout; + Process _child; vespalib::string _basename; bool _closed; bool _exited; diff --git a/fastos/src/tests/CMakeLists.txt b/fastos/src/tests/CMakeLists.txt index dce59650973..54341ff8e9a 100644 --- a/fastos/src/tests/CMakeLists.txt +++ b/fastos/src/tests/CMakeLists.txt @@ -1,10 +1,4 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(fastos_processtest_app TEST - SOURCES - processtest.cpp - DEPENDS - fastos -) vespa_add_executable(fastos_filetest_app TEST SOURCES filetest.cpp diff --git a/fastos/src/tests/filetest.cpp b/fastos/src/tests/filetest.cpp index 80045269c9e..a3470810e2e 100644 --- a/fastos/src/tests/filetest.cpp +++ b/fastos/src/tests/filetest.cpp @@ -40,8 +40,6 @@ bool createFile(const char* fileName, class FileTest : public BaseTest { -private: - virtual bool useProcessStarter() const override { return true; } public: const std::string srcDir = getenv("SOURCE_DIRECTORY") ? getenv("SOURCE_DIRECTORY") : "."; const std::string roFilename = srcDir + "/hello.txt"; diff --git a/fastos/src/tests/processtest.cpp b/fastos/src/tests/processtest.cpp deleted file mode 100644 index 204b87938de..00000000000 --- a/fastos/src/tests/processtest.cpp +++ /dev/null @@ -1,292 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "tests.h" -#include <vespa/fastos/process.h> -#include <thread> - -using namespace std::chrono_literals; -using namespace std::chrono; - -class MyListener : public FastOS_ProcessRedirectListener -{ -private: - MyListener(const MyListener&); - MyListener& operator=(const MyListener&); - - const char *_title; - int _receivedBytes; - -public: - static int _allocCount; - static int _successCount; - static int _failCount; - static std::mutex *_counterLock; - - MyListener (const char *title) - : _title(title), - _receivedBytes(0) - { - std::lock_guard<std::mutex> guard(*_counterLock); - _allocCount++; - } - - virtual ~MyListener () - { - bool isStdout = (strcmp(_title, "STDOUT") == 0); - - const int correctByteCount = 16; - - std::lock_guard<std::mutex> guard(*_counterLock); - if(_receivedBytes == (isStdout ? correctByteCount : 0)) - _successCount++; - else - _failCount++; - - _allocCount--; - } - - void OnReceiveData (const void *data, size_t length) override - { - _receivedBytes += length; - if(data != nullptr) - { - } - else - delete(this); - } -}; - -int MyListener::_allocCount = 0; -int MyListener::_successCount = 0; -int MyListener::_failCount = 0; -std::mutex *MyListener::_counterLock = nullptr; - -class ProcessTest : public BaseTest -{ -private: - bool useProcessStarter() const override { return true; } - bool useIPCHelper() const override { return true; } - ProcessTest(const ProcessTest&); - ProcessTest& operator=(const ProcessTest&); - int GetLastError () const { return errno; } - - // Flag which indicates whether an IPC message is received - // or not. - bool _gotMessage; - int _receivedMessages; - std::mutex *_counterLock; - bool _isChild; -public: - ProcessTest () - : _gotMessage(false), - _receivedMessages(0), - _counterLock(nullptr), - _isChild(true) - { - } - - void PollWaitTest () - { - TestHeader("PollWait Test"); - - FastOS_Process *xproc = new FastOS_Process("sort"); - - if(xproc->Create()) - { - int i; - for(i=0; i<10; i++) - { - bool stillRunning; - int returnCode; - - if(!xproc->PollWait(&returnCode, &stillRunning)) - { - Progress(false, "PollWait failure: %d", - GetLastError()); - break; - } - - if(i <= 5) - Progress(stillRunning, "StillRunning = %s", - stillRunning ? "true" : "false"); - - if(!stillRunning) - { - Progress(returnCode == 0, "Process exit code: %d", - returnCode); - break; - } - - if(i == 5) - { - // Make sort quit - xproc->WriteStdin(nullptr, 0); - } - - std::this_thread::sleep_for(1s); - } - - if(i == 10) - { - Progress(false, "Timeout"); - xproc->Kill(); - } - } - delete xproc; - - PrintSeparator(); - } - - void ProcessTests (bool doKill, bool stdinPre, bool waitKill) - { - const int numLoops = 100; - const int numEachTime = 40; - - MyListener::_counterLock = new std::mutex; - - char testHeader[200]; - strcpy(testHeader, "Process Test"); - if(doKill) - strcat(testHeader, " w/Kill"); - if(!stdinPre) - strcat(testHeader, " w/open stdin"); - if(waitKill) - strcat(testHeader, " w/Wait timeout"); - - TestHeader(testHeader); - - MyListener::_allocCount = 0; - MyListener::_successCount = 0; - MyListener::_failCount = 0; - - Progress(true, "Starting processes..."); - - for(int i=0; i<numLoops; i++) - { - FastOS_ProcessInterface *procs[numEachTime]; - - int j; - for(j=0; j<numEachTime; j++) - { - FastOS_ProcessInterface *xproc = - new FastOS_Process("sort", - new MyListener("STDOUT"), - new MyListener("STDERR")); - - if(xproc->Create()) - { - const char *str = "Peter\nPaul\nMary\n"; - - if(!waitKill && stdinPre) - { - xproc->WriteStdin(str, strlen(str)); - xproc->WriteStdin(nullptr, 0); - } - - if(doKill) - { - if(!xproc->Kill()) - Progress(false, "Kill failure %d", GetLastError()); - } - - if(!waitKill && !stdinPre) - { - xproc->WriteStdin(str, strlen(str)); - xproc->WriteStdin(nullptr, 0); - } - } - else - { - Progress(false, "Process.CreateWithShell failure %d", - GetLastError()); - delete xproc; - xproc = nullptr; - } - procs[j] = xproc; - } - - for(j=0; j<numEachTime; j++) - { - FastOS_ProcessInterface *xproc = procs[j]; - if(xproc == nullptr) - continue; - - int timeOut = -1; - if(waitKill) - timeOut = 1; - - steady_clock::time_point start = steady_clock::now(); - - int returnCode; - if(!xproc->Wait(&returnCode, timeOut)) - Progress(false, "Process.Wait failure %d", GetLastError()); - else - { - int checkReturnCode = 0; - if(doKill || waitKill) - checkReturnCode = FastOS_Process::KILL_EXITCODE; - if(returnCode != checkReturnCode) - Progress(false, "returnCode = %d", returnCode); - } - - if (waitKill) { - nanoseconds elapsed = steady_clock::now() - start; - if((elapsed < 900ms) || - (elapsed > 3500ms)) - { - Progress(false, "WaitKill time = %d", duration_cast<milliseconds>(elapsed).count()); - } - } - - delete xproc; - - if(waitKill) - Progress(true, "Started %d processes", i * numEachTime + j + 1); - } - - if(!waitKill && ((i % 10) == 9)) - Progress(true, "Started %d processes", (i+1) * numEachTime); - - if(waitKill && (((i+1) * numEachTime) > 50)) - break; - } - - Progress(MyListener::_allocCount == 0, "MyListener alloc count = %d", MyListener::_allocCount); - - if (!doKill && !waitKill) { - Progress(MyListener::_successCount == (2 * numLoops * numEachTime), - "MyListener _successCount = %d", MyListener::_successCount); - - Progress(MyListener::_failCount == 0, - "MyListener _failCount = %d", MyListener::_failCount); - } - - delete MyListener::_counterLock; - MyListener::_counterLock = nullptr; - - PrintSeparator(); - } - - int Main () override - { - _isChild = false; - - printf("grep for the string '%s' to detect failures.\n\n", failString); - - PollWaitTest(); - ProcessTests(false, true, false); - ProcessTests(true, true, false); - ProcessTests(true, false, false); - ProcessTests(false, true, true); - - printf("END OF TEST (%s)\n", _argv[0]); - - return allWasOk() ? 0 : 1; - } -}; - -int main (int argc, char **argv) -{ - ProcessTest app; - setvbuf(stdout, nullptr, _IOLBF, 8192); - return app.Entry(argc, argv); -} diff --git a/fastos/src/vespa/fastos/CMakeLists.txt b/fastos/src/vespa/fastos/CMakeLists.txt index 466c4f775a4..623d931e999 100644 --- a/fastos/src/vespa/fastos/CMakeLists.txt +++ b/fastos/src/vespa/fastos/CMakeLists.txt @@ -6,13 +6,10 @@ vespa_add_library(fastos_objects OBJECT file.cpp file_rw_ops.cpp linux_file.cpp - process.cpp thread.cpp unix_app.cpp unix_dynamiclibrary.cpp unix_file.cpp - unix_ipc.cpp - unix_process.cpp unix_thread.cpp ) diff --git a/fastos/src/vespa/fastos/app.cpp b/fastos/src/vespa/fastos/app.cpp index 0482cfea616..94e467d341d 100644 --- a/fastos/src/vespa/fastos/app.cpp +++ b/fastos/src/vespa/fastos/app.cpp @@ -8,13 +8,10 @@ #include "app.h" #include "file.h" -#include "process.h" #include "thread.h" #include <cstring> #include <fcntl.h> -FastOS_ApplicationInterface *FastOS_ProcessInterface::_app = nullptr; - FastOS_ThreadPool *FastOS_ApplicationInterface::GetThreadPool () { return _threadPool; @@ -22,12 +19,9 @@ FastOS_ThreadPool *FastOS_ApplicationInterface::GetThreadPool () FastOS_ApplicationInterface::FastOS_ApplicationInterface() : _threadPool(nullptr), - _processList(nullptr), - _processListMutex(nullptr), _argc(0), _argv(nullptr) { - FastOS_ProcessInterface::_app = this; #ifdef __linux__ char * fadvise = getenv("VESPA_FADVISE_OPTIONS"); if (fadvise != nullptr) { @@ -51,7 +45,6 @@ bool FastOS_ApplicationInterface::Init () if (PreThreadInit()) { if (FastOS_Thread::InitializeClass()) { if (FastOS_File::InitializeClass()) { - _processListMutex = new std::mutex; _threadPool = new FastOS_ThreadPool(128 * 1024); rc = true; } else @@ -72,12 +65,6 @@ void FastOS_ApplicationInterface::Cleanup () delete _threadPool; _threadPool = nullptr; } - - if(_processListMutex != nullptr) { - delete _processListMutex; - _processListMutex = nullptr; - } - FastOS_File::CleanupClass(); FastOS_Thread::CleanupClass(); } @@ -97,44 +84,3 @@ int FastOS_ApplicationInterface::Entry (int argc, char **argv) return rc; } - -void -FastOS_ApplicationInterface::AddChildProcess (FastOS_ProcessInterface *node) -{ - node->_prev = nullptr; - node->_next = _processList; - - if(_processList != nullptr) - _processList->_prev = node; - - _processList = node; -} - -void -FastOS_ApplicationInterface::RemoveChildProcess (FastOS_ProcessInterface *node) -{ - if(node->_prev) - node->_prev->_next = node->_next; - else - _processList = node->_next; - - if(node->_next) - { - node->_next->_prev = node->_prev; - node->_next = nullptr; - } - - if(node->_prev != nullptr) - node->_prev = nullptr; -} - -bool -FastOS_ApplicationInterface::useProcessStarter() const -{ - return false; -} -bool -FastOS_ApplicationInterface::useIPCHelper() const -{ - return useProcessStarter(); -} diff --git a/fastos/src/vespa/fastos/app.h b/fastos/src/vespa/fastos/app.h index 8d6a0b5ecbb..6aa7a9346ed 100644 --- a/fastos/src/vespa/fastos/app.h +++ b/fastos/src/vespa/fastos/app.h @@ -12,7 +12,6 @@ #include <vespa/fastos/types.h> -class FastOS_ProcessInterface; class FastOS_ThreadPool; #include <mutex> @@ -132,18 +131,7 @@ private: FastOS_ApplicationInterface& operator=(const FastOS_ApplicationInterface&); protected: - /** - * - * Indicate if a process starter is going to be used. - * Only override this one if you are going to start other processes. - * @return true if you are going to use a process starter. - */ - virtual bool useProcessStarter() const; - virtual bool useIPCHelper() const; - FastOS_ThreadPool *_threadPool; - FastOS_ProcessInterface *_processList; - std::mutex *_processListMutex; virtual bool PreThreadInit () { return true; } @@ -195,11 +183,6 @@ public: */ virtual void Cleanup (); - void AddChildProcess (FastOS_ProcessInterface *node); - void RemoveChildProcess (FastOS_ProcessInterface *node); - std::unique_lock<std::mutex> getProcessGuard() { return std::unique_lock<std::mutex>(*_processListMutex); } - FastOS_ProcessInterface *GetProcessList () { return _processList; } - FastOS_ThreadPool *GetThreadPool (); }; diff --git a/fastos/src/vespa/fastos/process.cpp b/fastos/src/vespa/fastos/process.cpp deleted file mode 100644 index 8e4f4afdc98..00000000000 --- a/fastos/src/vespa/fastos/process.cpp +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "process.h" - -FastOS_ProcessInterface::FastOS_ProcessInterface (const char *cmdLine, - FastOS_ProcessRedirectListener *stdoutListener, - FastOS_ProcessRedirectListener *stderrListener) : - _cmdLine(cmdLine), - _stdoutListener(stdoutListener), - _stderrListener(stderrListener), - _next(nullptr), - _prev(nullptr) -{ -} - -FastOS_ProcessInterface::~FastOS_ProcessInterface () = default; diff --git a/fastos/src/vespa/fastos/process.h b/fastos/src/vespa/fastos/process.h deleted file mode 100644 index f520fcd30f8..00000000000 --- a/fastos/src/vespa/fastos/process.h +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -//************************************************************************ -/** - * @file - * Class definitions for FastOS_ProcessInterface and - * FastOS_ProcessRedirectListener. - * - * @author Oivind H. Danielsen - */ - -#pragma once - -#include "types.h" -#include <cstddef> -#include <string> - -/** - * This class serves as a sink for redirected (piped) output from - * subclasses of @ref FastOS_ProcessInterface. - */ -class FastOS_ProcessRedirectListener -{ -public: - /** - * This method is called when new data is available from the - * process. Subclass this method to process the data. - * You should assume that any thread can invoke this method. - * For convenience the data buffer is always zero- terminated - * (static_cast<uint8_t>(data[length]) = '\\0'). - * When the pipe closes, the method is invoked with data = nullptr - * and length = 0. - * @param data Pointer to data - * @param length Length of data block in bytes - */ - virtual void OnReceiveData (const void *data, size_t length) = 0; - - virtual ~FastOS_ProcessRedirectListener () {} -}; - -class FastOS_ThreadPool; -class FastOS_ApplicationInterface; - -/** - * This class can start a process, redirect standard input, output - * and error streams, kill process, wait for process to exit - * and send IPC messages to the process. - */ -class FastOS_ProcessInterface -{ -protected: - - std::string _cmdLine; - FastOS_ProcessRedirectListener *_stdoutListener; - FastOS_ProcessRedirectListener *_stderrListener; - -public: - FastOS_ProcessInterface *_next, *_prev; - static FastOS_ApplicationInterface *_app; - - enum Constants - { - KILL_EXITCODE = 65535, /* Process killed or failed */ - CONSTEND - }; - - /** - * Constructor. Does not start the process, use @ref Create or - * @ref CreateWithShell to actually start the process. - * @param cmdLine Command line - * @param stdoutListener non-nullptr to redirect stdout - * @param stderrListener non-nullptr to redirect stderr - * @param bufferSize Size of redirect buffers - */ - FastOS_ProcessInterface (const char *cmdLine, - FastOS_ProcessRedirectListener *stdoutListener = nullptr, - FastOS_ProcessRedirectListener *stderrListener = nullptr); - - FastOS_ProcessInterface(const FastOS_ProcessInterface&) = delete; - FastOS_ProcessInterface &operator=(const FastOS_ProcessInterface &) = delete; - /** - * Destructor. - * If @ref Wait has not been called yet, it is called here. - */ - virtual ~FastOS_ProcessInterface (); - - /** - * Create and start the process. If your command line includes - * commands specific to the shell use @ref CreateWithShell instead. - * - * IPC communication currently only supports direct parent/child - * relationships. If you launch a FastOS application through - * the shell or some other script/process, the FastOS application - * might not be a direct child of your process and IPC communication - * will not work (the rest will work ok, though). - * - * This limitation might be removed in the future. - * @return Boolean success / failure - */ - virtual bool Create() = 0; - - /** - * Create and start the process using the default OS shell - * (UNIX: /bin/sh). - * - * IPC communication currently only supports direct parent/child - * relationships. If you launch a FastOS application through - * the shell or some other script/process, the FastOS application - * might not be a direct child of your process and IPC communication - * will not work (the rest will work ok, though). - * - * This limitation might be removed in the future. - * @return Boolean success / failure - */ - virtual bool CreateWithShell() = 0; - - /** - * If you are redirecting the standard input stream of the process, - * use this method to write data. To close the input stream, - * invoke @ref WriteStdin with data=nullptr. If the input stream - * is not redirected, @ref WriteStdin will fail. - * @param data Pointer to data - * @param length Length of data block in bytes - * @return Boolean success / failure - */ - virtual bool WriteStdin (const void *data, size_t length) = 0; - - /** - * Terminate the process. !!IMPORTANT LIMITATION!!: There is no guarantee - * that child processes (of the process to be killed) will be killed - * as well. - * @return Boolean success / failure - */ - virtual bool Kill () = 0; - - /** - * Wait for the process to finish / terminate. This is called - * automatically by the destructor, but it is recommended that - * it is called as early as possible to free up resources. - * @param returnCode Pointer to int which will receive - * the process return code. - * @param timeOutSeconds Number of seconds to wait before - * the process is violently killed. - * -1 = infinite wait / no timeout - * @return Boolean success / failure - */ - virtual bool Wait (int *returnCode, int timeOutSeconds = -1) = 0; - - /** - * Poll version of @ref Wait. - * This is basically @ref Wait with a timeout of 0 seconds. - * The process is not killed if the "timeout" expires. - * A boolean value, stillRunning, is set to indicate whether - * the process is still running or not. - * There is no need to invoke @ref Wait if @ref PollWait - * indicates that the process is finished. - * @param returnCode Pointer to int which will receive - * the process return code. - * @param stillRunning Pointer to boolean value which will - * be set to indicate whether the - * process is still running or not. - * @return Boolean success / failure - */ - virtual bool PollWait (int *returnCode, bool *stillRunning) = 0; - - /** - * Get process identification number. - * @return Process id - */ - virtual unsigned int GetProcessId() = 0; - - /** - * Get command line string. - * @return Command line string - */ - const char *GetCommandLine () const { return _cmdLine.c_str(); } -}; - -#include <vespa/fastos/unix_process.h> -typedef FastOS_UNIX_Process FASTOS_PREFIX(Process); - diff --git a/fastos/src/vespa/fastos/unix_app.cpp b/fastos/src/vespa/fastos/unix_app.cpp index b0baf3990ad..e94525fff22 100644 --- a/fastos/src/vespa/fastos/unix_app.cpp +++ b/fastos/src/vespa/fastos/unix_app.cpp @@ -9,19 +9,12 @@ #include "app.h" #include "time.h" -#include "process.h" -#include "unix_ipc.h" #include <unistd.h> #include <csignal> #include <getopt.h> -FastOS_UNIX_Application::FastOS_UNIX_Application () - : _processStarter(), - _ipcHelper(nullptr) -{ -} - +FastOS_UNIX_Application::FastOS_UNIX_Application() = default; FastOS_UNIX_Application::~FastOS_UNIX_Application() = default; extern "C" @@ -72,10 +65,6 @@ bool FastOS_UNIX_Application::PreThreadInit () sigemptyset(&act.sa_mask); act.sa_flags = 0; sigaction(SIGPIPE, &act, nullptr); - - if (useProcessStarter()) { - _processStarter = std::make_unique<FastOS_UNIX_ProcessStarter>(this); - } } else { rc = false; fprintf(stderr, "FastOS_ApplicationInterface::PreThreadInit failed\n"); @@ -89,13 +78,6 @@ bool FastOS_UNIX_Application::Init () if(FastOS_ApplicationInterface::Init()) { - int ipcDescriptor = -1; - - if (useIPCHelper()) { - _ipcHelper = new FastOS_UNIX_IPCHelper(this, ipcDescriptor); - GetThreadPool()->NewThread(_ipcHelper); - } - rc = true; } @@ -104,38 +86,5 @@ bool FastOS_UNIX_Application::Init () void FastOS_UNIX_Application::Cleanup () { - if(_ipcHelper != nullptr) - _ipcHelper->Exit(); - - if (_processStarter) { - { - std::unique_lock<std::mutex> guard; - if (_processListMutex) { - guard = getProcessGuard(); - } - } - _processStarter.reset(); - } - FastOS_ApplicationInterface::Cleanup(); } - -FastOS_UNIX_ProcessStarter * -FastOS_UNIX_Application::GetProcessStarter () -{ - return _processStarter.get(); -} - -void FastOS_UNIX_Application:: -AddToIPCComm (FastOS_UNIX_Process *process) -{ - if(_ipcHelper != nullptr) - _ipcHelper->AddProcess(process); -} - -void FastOS_UNIX_Application:: -RemoveFromIPCComm (FastOS_UNIX_Process *process) -{ - if(_ipcHelper != nullptr) - _ipcHelper->RemoveProcess(process); -} diff --git a/fastos/src/vespa/fastos/unix_app.h b/fastos/src/vespa/fastos/unix_app.h index 5bb6f14ad26..5e076f4482b 100644 --- a/fastos/src/vespa/fastos/unix_app.h +++ b/fastos/src/vespa/fastos/unix_app.h @@ -13,10 +13,6 @@ #include "app.h" #include <memory> -class FastOS_UNIX_ProcessStarter; -class FastOS_UNIX_IPCHelper; -class FastOS_UNIX_Process; - /** * This is the generic UNIX implementation of @ref FastOS_ApplicationInterface */ @@ -26,9 +22,6 @@ private: FastOS_UNIX_Application(const FastOS_UNIX_Application&); FastOS_UNIX_Application& operator=(const FastOS_UNIX_Application&); - std::unique_ptr<FastOS_UNIX_ProcessStarter> _processStarter; - FastOS_UNIX_IPCHelper *_ipcHelper; - protected: bool PreThreadInit () override; public: @@ -71,11 +64,6 @@ public: */ static void resetOptIndex(int OptionIndex); - FastOS_UNIX_ProcessStarter *GetProcessStarter (); bool Init () override; void Cleanup () override; - void AddToIPCComm (FastOS_UNIX_Process *process); - void RemoveFromIPCComm (FastOS_UNIX_Process *process); }; - - diff --git a/fastos/src/vespa/fastos/unix_ipc.cpp b/fastos/src/vespa/fastos/unix_ipc.cpp deleted file mode 100644 index b75d07751fa..00000000000 --- a/fastos/src/vespa/fastos/unix_ipc.cpp +++ /dev/null @@ -1,547 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "unix_ipc.h" -#include "ringbuffer.h" -#include <cassert> -#include <cstring> -#include <cstdlib> -#include <unistd.h> -#include <fcntl.h> -#include <memory> -#include <future> - -FastOS_UNIX_IPCHelper:: -FastOS_UNIX_IPCHelper (FastOS_ApplicationInterface *app, int) - : _lock(), - _exitFlag(false), - _app(app) -{ - _wakeupPipe[0] = -1; - _wakeupPipe[1] = -1; - - if(pipe(_wakeupPipe) != 0) { - perror("pipe wakeuppipe"); - std::_Exit(1); - } else { - SetBlocking(_wakeupPipe[0], false); - SetBlocking(_wakeupPipe[1], true); - } -} - -FastOS_UNIX_IPCHelper::~FastOS_UNIX_IPCHelper () -{ - if(_wakeupPipe[0] != -1) { - close(_wakeupPipe[0]); - } - if(_wakeupPipe[1] != -1) { - close(_wakeupPipe[1]); - } -} - - -bool FastOS_UNIX_IPCHelper:: -DoWrite(FastOS_UNIX_Process::DescriptorHandle &desc) -{ - bool rc = true; - FastOS_RingBuffer *buffer = desc._writeBuffer.get(); - - auto bufferGuard = buffer->getGuard(); - int writeBytes = buffer->GetReadSpace(); - if(writeBytes > 0) - { - int bytesWritten; - do - { - bytesWritten = write(desc._fd, - buffer->GetReadPtr(), - writeBytes); - } while(bytesWritten < 0 && errno == EINTR); - - if(bytesWritten > 0) - buffer->Consume(bytesWritten); - else if(bytesWritten < 0) - { - desc.CloseHandle(); - perror("FastOS_UNIX_IPCHelper::DoWrite"); - rc = false; - } - else if(bytesWritten == 0) - desc.CloseHandle(); - } - return rc; -} - -bool FastOS_UNIX_IPCHelper:: -DoRead (FastOS_UNIX_Process::DescriptorHandle &desc) -{ - bool rc = true; - - FastOS_RingBuffer *buffer = desc._readBuffer.get(); - - auto bufferGuard = buffer->getGuard(); - int readBytes = buffer->GetWriteSpace(); - if(readBytes > 0) { - int bytesRead; - do { - bytesRead = read(desc._fd, buffer->GetWritePtr(), readBytes); - } while(bytesRead < 0 && errno == EINTR); - - if (bytesRead > 0) { - buffer->Produce(bytesRead); - } else if(bytesRead < 0) { - desc.CloseHandle(); - perror("FastOS_UNIX_IPCHelper::DoRead"); - rc = false; - } else if(bytesRead == 0) { - desc.CloseHandle(); - } - } - - return rc; -} - -bool FastOS_UNIX_IPCHelper:: -SetBlocking (int fileDescriptor, bool doBlock) -{ - bool rc=false; - - int flags = fcntl(fileDescriptor, F_GETFL, nullptr); - if (flags != -1) - { - if(doBlock) - flags &= ~O_NONBLOCK; - else - flags |= O_NONBLOCK; - rc = (fcntl(fileDescriptor, F_SETFL, flags) != -1); - } - return rc; -} - -void FastOS_UNIX_IPCHelper:: -BuildPollCheck(bool isRead, int filedes, - FastOS_RingBuffer *buffer, bool *check) -{ - if(buffer == nullptr || - filedes < 0 || - buffer->GetCloseFlag()) { - *check = false; - return; - } - - bool setIt = false; - if(isRead) - setIt = (buffer->GetWriteSpace() > 0); - else - setIt = (buffer->GetReadSpace() > 0); - *check = setIt; -} - - -void FastOS_UNIX_IPCHelper:: -PerformAsyncIO() -{ - FastOS_ProcessInterface *node; - for(node = _app->GetProcessList(); node != nullptr; node = node->_next) - { - FastOS_UNIX_Process *xproc = static_cast<FastOS_UNIX_Process *>(node); - - for(int type=0; type < int(FastOS_UNIX_Process::TYPE_READCOUNT); type++) - { - FastOS_UNIX_Process::DescriptorType type_ = - FastOS_UNIX_Process::DescriptorType(type); - FastOS_UNIX_Process::DescriptorHandle &desc = - xproc->GetDescriptorHandle(type_); - if (desc._canRead) - (void) DoRead(desc); - if (desc._canWrite) - (void) DoWrite(desc); - } - } -} - - -void FastOS_UNIX_IPCHelper:: -BuildPollChecks() -{ - FastOS_ProcessInterface *node; - for(node = _app->GetProcessList(); node != nullptr; node = node->_next) - { - FastOS_UNIX_Process *xproc = static_cast<FastOS_UNIX_Process *>(node); - - for(int type=0; type < int(FastOS_UNIX_Process::TYPE_READCOUNT); type++) - { - FastOS_UNIX_Process::DescriptorType type_ = - FastOS_UNIX_Process::DescriptorType(type); - FastOS_UNIX_Process::DescriptorHandle &desc = - xproc->GetDescriptorHandle(type_); - BuildPollCheck(false, desc._fd, desc._writeBuffer.get(), &desc._wantWrite); - BuildPollCheck(true, desc._fd, desc._readBuffer.get(), &desc._wantRead); - } - } -} - - -static pollfd * -__attribute__((__noinline__)) -ResizePollArray(pollfd **fds, unsigned int *allocnfds) -{ - pollfd *newfds; - unsigned int newallocnfds; - - if (*allocnfds == 0) - newallocnfds = 16; - else - newallocnfds = *allocnfds * 2; - newfds = static_cast<pollfd *>(malloc(newallocnfds * sizeof(pollfd))); - assert(newfds != nullptr); - - if (*allocnfds > 0) - memcpy(newfds, *fds, sizeof(pollfd) * *allocnfds); - - if (*fds != nullptr) - free(*fds); - - *fds = newfds; - newfds += *allocnfds; - *allocnfds = newallocnfds; - return newfds; -} - -void -FastOS_UNIX_IPCHelper:: -BuildPollArray(pollfd **fds, unsigned int *nfds, unsigned int *allocnfds) -{ - FastOS_ProcessInterface *node; - pollfd *rfds; - const pollfd *rfdsEnd; - int pollIdx; - - rfds = *fds; - rfdsEnd = *fds + *allocnfds; - - if (rfds >= rfdsEnd) { - rfds = ResizePollArray(fds, - allocnfds); - rfdsEnd = *fds + *allocnfds; - } - rfds->fd = _wakeupPipe[0]; - rfds->events = POLLIN; - rfds->revents = 0; - rfds++; - pollIdx = 1; - for(node = _app->GetProcessList(); node != nullptr; node = node->_next) - { - FastOS_UNIX_Process *xproc = static_cast<FastOS_UNIX_Process *>(node); - - for(int type=0; type < int(FastOS_UNIX_Process::TYPE_READCOUNT); type++) - { - FastOS_UNIX_Process::DescriptorType type_ = - FastOS_UNIX_Process::DescriptorType(type); - FastOS_UNIX_Process::DescriptorHandle &desc = - xproc->GetDescriptorHandle(type_); - - if (desc._fd >= 0 && - (desc._wantRead || desc._wantWrite)) { - if (rfds >= rfdsEnd) { - rfds = ResizePollArray(fds, - allocnfds); - rfdsEnd = *fds + *allocnfds; - } - rfds->fd = desc._fd; - rfds->events = 0; - if (desc._wantRead) - rfds->events |= POLLRDNORM; - if (desc._wantWrite) - rfds->events |= POLLWRNORM; - rfds->revents = 0; - desc._pollIdx = pollIdx; - rfds++; - pollIdx++; - } else { - desc._pollIdx = -1; - desc._canRead = false; - desc._canWrite = false; - } - } - } - - *nfds = rfds - *fds; -} - - -bool -FastOS_UNIX_IPCHelper:: -SavePollArray(pollfd *fds, unsigned int nfds) -{ - FastOS_ProcessInterface *node; - - for(node = _app->GetProcessList(); node != nullptr; node = node->_next) - { - FastOS_UNIX_Process *xproc = static_cast<FastOS_UNIX_Process *>(node); - - for(int type=0; type < int(FastOS_UNIX_Process::TYPE_READCOUNT); type++) - { - FastOS_UNIX_Process::DescriptorType type_ = - FastOS_UNIX_Process::DescriptorType(type); - FastOS_UNIX_Process::DescriptorHandle &desc = - xproc->GetDescriptorHandle(type_); - - if (desc._fd >= 0 && - static_cast<unsigned int>(desc._pollIdx) < nfds) { - int revents = fds[desc._pollIdx].revents; - - if (desc._wantRead && - (revents & - (POLLIN | POLLRDNORM | POLLERR | POLLHUP | POLLNVAL)) != 0) - desc._canRead = true; - else - desc._canRead = false; - if (desc._wantWrite && - (revents & - (POLLOUT | POLLWRNORM | POLLWRBAND | POLLERR | POLLHUP | - POLLNVAL)) != 0) - desc._canWrite = true; - else - desc._canWrite = false; - } - } - } - - if ((fds[0].revents & (POLLIN | POLLERR | POLLHUP)) != 0) - return true; - else - return false; -} - - -void FastOS_UNIX_IPCHelper:: -RemoveClosingProcesses() -{ - // We assume that not updating maxFD isn't harmless. - - FastOS_ProcessInterface *node, *next; - - for(node = _app->GetProcessList(); node != nullptr; node = next) - { - int type; - - next = node->_next; - FastOS_UNIX_Process *xproc = static_cast<FastOS_UNIX_Process *>(node); - - bool stillBusy = false; - if(!xproc->GetKillFlag()) - for(type=0; type < FastOS_UNIX_Process::TYPE_READCOUNT; type++) - { - FastOS_UNIX_Process::DescriptorType type_; - - type_ = static_cast<FastOS_UNIX_Process::DescriptorType>(type); - - FastOS_UNIX_Process::DescriptorHandle &desc = - xproc->GetDescriptorHandle(type_); - - if (desc._fd != -1) - { - if((type_ == FastOS_UNIX_Process::TYPE_STDOUT) || - (type_ == FastOS_UNIX_Process::TYPE_STDERR) || - desc._wantWrite) - { - // We still want to use this socket. - // Make sure we don't close the socket yet. - stillBusy = true; - break; - } - } - } - - if(!stillBusy) - { - if (xproc->_closing) { - // We already have the process lock at this point, - // so modifying the list is safe. - _app->RemoveChildProcess(node); - - for(type=0; type < FastOS_UNIX_Process::TYPE_READCOUNT; type++) - { - FastOS_UNIX_Process::DescriptorHandle &desc = - xproc->GetDescriptorHandle(FastOS_UNIX_Process::DescriptorType(type)); - if(desc._fd != -1) - { - // No more select on this one. - // We already know wantWrite is not set - if (desc._wantRead) - desc._wantRead = false; - } - } - - // The process destructor can now proceed - auto closingPromise(std::move(xproc->_closing)); - closingPromise->set_value(); - } - } - } -} - - -void FastOS_UNIX_IPCHelper:: -Run(FastOS_ThreadInterface *thisThread, void *arg) -{ - (void)arg; - (void)thisThread; - - FastOS_ProcessInterface *node; - pollfd *fds; - unsigned int nfds; - unsigned int allocnfds; - - fds = nullptr; - nfds = 0; - allocnfds = 0; - for(;;) - { - // Deliver messages to from child processes and parent. - { - auto guard = _app->getProcessGuard(); - for(node = _app->GetProcessList(); node != nullptr; node = node->_next) - { - FastOS_UNIX_Process *xproc = static_cast<FastOS_UNIX_Process *>(node); - PipeData(xproc, FastOS_UNIX_Process::TYPE_STDOUT); - PipeData(xproc, FastOS_UNIX_Process::TYPE_STDERR); - } - - // Setup file descriptor sets for the next select() call - BuildPollChecks(); - - // Close and signal closing processes - RemoveClosingProcesses(); - - BuildPollArray(&fds, &nfds, &allocnfds); - } - bool exitFlag = false; - { - std::lock_guard<std::mutex> guard(_lock); - exitFlag = _exitFlag; - } - if (exitFlag) - { - break; - } - - for (;;) - { - int pollRc = - poll(fds, nfds, -1); - - if(pollRc == -1) - { - int wasErrno = errno; - - if(wasErrno == EINTR) - { - continue; - } - - perror("FastOS_UNIX_IPCHelper::RunAsync select failure"); - printf("errno = %d\n", wasErrno); - for(unsigned int i = 0; i < nfds; i++) - { - if ((fds[i].events & POLLIN) != 0) - printf("Read %d\n", fds[i].fd); - if ((fds[i].events & POLLOUT) != 0) - printf("Write %d\n", fds[i].fd); - } - std::_Exit(1); - } else { - break; - } - } - - bool woken = false; - { - auto guard = _app->getProcessGuard(); - woken = SavePollArray(fds, nfds); - // Do actual IO (based on file descriptor sets and buffer contents) - PerformAsyncIO(); - } - - // Did someone want to wake us up from the poll() call? - if (woken) { - char dummy; - ssize_t nbrfp = read(_wakeupPipe[0], &dummy, 1); - if (nbrfp != 1) { - perror("FastOS_UNIX_IPCHelper wakeupPipe read failed"); - } - } - } - free(fds); - - delete this; -} - -void -FastOS_UNIX_IPCHelper::NotifyProcessListChange () -{ - char dummy = 'x'; - ssize_t nbwtp = write(_wakeupPipe[1], &dummy, 1); - if (nbwtp != 1) { - perror("FastOS_UNIX_IPCHelper: write to wakeupPipe failed"); - } -} - -void -FastOS_UNIX_IPCHelper::Exit () -{ - std::lock_guard<std::mutex> guard(_lock); - _exitFlag = true; - NotifyProcessListChange(); -} - -void -FastOS_UNIX_IPCHelper::AddProcess (FastOS_UNIX_Process *xproc) -{ - bool newStream = false; - for(int type=0; type < int(FastOS_UNIX_Process::TYPE_READCOUNT); type++) - { - FastOS_UNIX_Process::DescriptorType type_ = FastOS_UNIX_Process::DescriptorType(type); - FastOS_UNIX_Process::DescriptorHandle &desc = xproc->GetDescriptorHandle(type_); - - if (desc._fd != -1) { - newStream = true; - SetBlocking(desc._fd, false); - } - } - if(newStream) - NotifyProcessListChange(); -} - -void -FastOS_UNIX_IPCHelper::RemoveProcess (FastOS_UNIX_Process *xproc) -{ - auto closePromise = std::make_unique<std::promise<void>>(); - auto closeFuture = closePromise->get_future(); - xproc->_closing = std::move(closePromise); - NotifyProcessListChange(); - closeFuture.wait(); -} - -void -FastOS_UNIX_IPCHelper::PipeData(FastOS_UNIX_Process *process, FastOS_UNIX_Process::DescriptorType type) -{ - FastOS_UNIX_Process::DescriptorHandle &desc = process->GetDescriptorHandle(type); - FastOS_RingBuffer *buffer = desc._readBuffer.get(); - if(buffer == nullptr) - return; - - FastOS_ProcessRedirectListener *listener = process->GetListener(type); - if(listener == nullptr) - return; - - auto bufferGuard = buffer->getGuard(); - - unsigned int readSpace; - while((readSpace = buffer->GetReadSpace()) > 0) { - listener->OnReceiveData(buffer->GetReadPtr(), size_t(readSpace)); - buffer->Consume(readSpace); - } - - if(buffer->GetCloseFlag()) - process->CloseListener(type); -} diff --git a/fastos/src/vespa/fastos/unix_ipc.h b/fastos/src/vespa/fastos/unix_ipc.h deleted file mode 100644 index 13e92411011..00000000000 --- a/fastos/src/vespa/fastos/unix_ipc.h +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "app.h" -#include "process.h" -#include "thread.h" -#include <poll.h> - -class FastOS_RingBuffer; - -class FastOS_UNIX_IPCHelper : public FastOS_Runnable -{ -private: - FastOS_UNIX_IPCHelper(const FastOS_UNIX_IPCHelper&); - FastOS_UNIX_IPCHelper& operator=(const FastOS_UNIX_IPCHelper&); - -protected: - std::mutex _lock; - volatile bool _exitFlag; - FastOS_ApplicationInterface *_app; - - int _wakeupPipe[2]; - - bool DoWrite (FastOS_UNIX_Process::DescriptorHandle &desc); - bool DoRead (FastOS_UNIX_Process::DescriptorHandle &desc); - bool SetBlocking (int fileDescriptor, bool doBlock); - void BuildPollCheck (bool isRead, int filedes, FastOS_RingBuffer *buffer, bool *check); - void BuildPollArray(pollfd **fds, unsigned int *nfds, unsigned int *allocnfds); - bool SavePollArray(pollfd *fds, unsigned int nfds); - void PerformAsyncIO (); - void BuildPollChecks(); - void PipeData (FastOS_UNIX_Process *process, FastOS_UNIX_Process::DescriptorType type); - void RemoveClosingProcesses(); - -public: - FastOS_UNIX_IPCHelper (FastOS_ApplicationInterface *app, int appDescriptor); - ~FastOS_UNIX_IPCHelper (); - void Run (FastOS_ThreadInterface *thisThread, void *arg) override; - void NotifyProcessListChange (); - void AddProcess (FastOS_UNIX_Process *xproc); - void RemoveProcess (FastOS_UNIX_Process *xproc); - void Exit (); -}; diff --git a/fastos/src/vespa/fastos/unix_process.cpp b/fastos/src/vespa/fastos/unix_process.cpp deleted file mode 100644 index cd29108c3d6..00000000000 --- a/fastos/src/vespa/fastos/unix_process.cpp +++ /dev/null @@ -1,985 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "process.h" -#include "unix_ipc.h" -#include "ringbuffer.h" -#include <vector> -#include <cstring> -#include <cstdlib> -#include <unistd.h> -#include <fcntl.h> -#include <sys/wait.h> -#ifndef __linux__ -#include <signal.h> -#endif -#include <thread> - - -extern "C" -{ -extern char **environ; -} - - -using namespace std::chrono_literals; -using namespace std::chrono; - -static pid_t safe_fork () -{ - pid_t pid; - int retry = 1; - while((pid = fork()) == -1 && errno == EAGAIN) { - sleep(retry); - if (retry < 4) retry *= 2; - } - return pid; -} - -static int -normalizedWaitStatus(int status) -{ - if (WIFEXITED(status)) - return WEXITSTATUS(status); - else - return (0x80000000 | status); -} - - -// The actual process launched in the proxy process -class FastOS_UNIX_RealProcess -{ -private: - FastOS_UNIX_RealProcess(const FastOS_UNIX_RealProcess&); - FastOS_UNIX_RealProcess& operator=(const FastOS_UNIX_RealProcess&); - -public: - enum - { - STREAM_STDIN = (1 << 0), - STREAM_STDOUT = (1 << 1), - STREAM_STDERR = (1 << 2), - EXEC_SHELL = (1 << 3) - }; - -private: - pid_t _pid; - bool _terse; // Set if using direct fork (bypassing proxy process) - int _streamMask; - - int _stdinDes[2]; - int _stdoutDes[2]; - int _stderrDes[2]; - int _handshakeDes[2]; - std::string _runDir; - std::string _stdoutRedirName; - std::string _stderrRedirName; - const char *_path; - std::vector<char> _pathProgBuf; - - void CloseDescriptor(int fd); - void CloseAndResetDescriptor(int *fd); - void CloseDescriptors(); - -public: - void SetRunDir(const char * runDir) { _runDir = runDir; } - - int HandoverStdinDescriptor() { - int ret = _stdinDes[1]; - _stdinDes[1] = -1; - return ret; - } - - int HandoverStdoutDescriptor() { - int ret = _stdoutDes[0]; - _stdoutDes[0] = -1; - return ret; - } - - int HandoverStderrDescriptor() { - int ret = _stderrDes[0]; - _stderrDes[0] = -1; - return ret; - } - - FastOS_UNIX_RealProcess *_prev, *_next; - - FastOS_UNIX_RealProcess (int streamMask); - ~FastOS_UNIX_RealProcess(); - pid_t GetProcessID() const { return _pid; } - - bool IsStdinPiped() const { - return (_streamMask & STREAM_STDIN ) != 0; - } - - bool IsStdoutPiped() const { - return (_streamMask & STREAM_STDOUT) != 0; - } - - bool IsStderrPiped() const { - return (_streamMask & STREAM_STDERR) != 0; - } - - bool IsUsingShell() const { - return (_streamMask & EXEC_SHELL) != 0; - } - - void SetStdoutRedirName(const char *stdoutRedirName) { - _stdoutRedirName = stdoutRedirName; - } - - void SetStderrRedirName(const char *stderrRedirName) { - _stderrRedirName = stderrRedirName; - } - - void PrepareExecVPE (const char *prog); - - void - ExecVPE (const char *prog, - char *const args[], - char *const env[]); - - static bool IsWhiteSpace (char c); - - static const char * - NextArgument (const char *p, - const char **endArg, - int *length = nullptr); - - static int CountArguments (const char *commandLine); - - void - RedirOut(const std::string & filename, - int targetfd, - int exitCodeOnFailure); - - bool - ForkAndExec(const char *command, - char **environmentVariables, - FastOS_UNIX_Process *process); - - bool Setup(); - void SetTerse() { _terse = true; } - ssize_t HandshakeRead(void *buf, size_t len); - void HandshakeWrite(int val); -}; - - -void -FastOS_UNIX_RealProcess::CloseDescriptor(int fd) -{ - close(fd); -} - - -void -FastOS_UNIX_RealProcess::CloseAndResetDescriptor(int *fd) -{ - if (*fd == -1) - return; - CloseDescriptor(*fd); - *fd = -1; -} - - -void -FastOS_UNIX_RealProcess::CloseDescriptors() -{ - CloseAndResetDescriptor(&_stdinDes[0]); - CloseAndResetDescriptor(&_stdinDes[1]); - CloseAndResetDescriptor(&_stdoutDes[0]); - CloseAndResetDescriptor(&_stdoutDes[1]); - CloseAndResetDescriptor(&_stderrDes[0]); - CloseAndResetDescriptor(&_stderrDes[1]); - CloseAndResetDescriptor(&_handshakeDes[0]); - CloseAndResetDescriptor(&_handshakeDes[1]); -} - - -FastOS_UNIX_RealProcess::FastOS_UNIX_RealProcess(int streamMask) - : _pid(-1), - _terse(false), - _streamMask(streamMask), - _runDir(), - _stdoutRedirName(), - _stderrRedirName(), - _path(nullptr), - _pathProgBuf(), - _prev(nullptr), - _next(nullptr) -{ - _stdinDes[0] = _stdinDes[1] = -1; - _stdoutDes[0] = _stdoutDes[1] = -1; - _stderrDes[0] = _stderrDes[1] = -1; - _handshakeDes[0] = _handshakeDes[1] = -1; -} - - -FastOS_UNIX_RealProcess::~FastOS_UNIX_RealProcess() -{ - CloseDescriptors(); -} - - -void -FastOS_UNIX_RealProcess::PrepareExecVPE(const char *prog) -{ - const char *path = nullptr; - - char defaultPath[] = ":/usr/ucb:/bin:/usr/bin"; - - if (strchr(prog, '/') != nullptr) { - path = ""; - } else { - path = getenv("PATH"); - if (path == nullptr) path = defaultPath; - } - _path = path; - _pathProgBuf.resize(strlen(prog) + 1 + strlen(path) + 1); -} - - -void -FastOS_UNIX_RealProcess::ExecVPE (const char *prog, - char *const args[], - char *const env[]) -{ - - char *fullPath = &_pathProgBuf[0]; - const char *path = _path; - - for(;;) - { - char *p; - for (p = fullPath; (*path != '\0') && (*path != ':'); path++) - *p++ = *path; - - if (p > fullPath) *p++ = '/'; - - strcpy(p, prog); - // fprintf(stdout, "Attempting execve [%s]\n", fullPath); - // fflush(stdout); - execve(fullPath, args, env); - - if ((errno == ENOEXEC) || - (errno == ENOMEM) || - (errno == E2BIG) || - (errno == ETXTBSY)) - break; - - if (*path == '\0') break; - path++; - } -} - - -bool -FastOS_UNIX_RealProcess::IsWhiteSpace (char c) -{ - return (c == ' ' || c == '\t'); -} - - -const char * -FastOS_UNIX_RealProcess::NextArgument (const char *p, - const char **endArg, - int *length) -{ - while(*p != '\0') - { - if (!IsWhiteSpace(*p)) { - char quoteChar = '\0'; - if ((*p == '\'') || (*p == '"')) { - quoteChar = *p; - p++; - } - - const char *nextArg = p; - - // Find the end of the argument. - for(;;) - { - if (*p == '\0') { - if (length != nullptr) - *length = p - nextArg; - break; - } - - if (quoteChar != '\0') { - if (*p == quoteChar) { - if (length != nullptr) - *length = p - nextArg; - p++; - break; - } - } - else - { - if (IsWhiteSpace(*p)) { - if (length != nullptr) - *length = p - nextArg; - break; - } - } - p++; - } - - *endArg = p; - return nextArg; - } - p++; - } - return nullptr; -} - - -int -FastOS_UNIX_RealProcess::CountArguments (const char *commandLine) -{ - int numArgs = 0; - const char *nextArg = commandLine; - while(NextArgument(nextArg, &nextArg)) - numArgs++; - - return numArgs; -} - - -void -FastOS_UNIX_RealProcess::RedirOut(const std::string & filename, - int targetfd, - int exitCodeOnFailure) -{ - if (filename.empty() || filename[0] != '>') - return; - - int newfd; - if (filename[1] == '>') { - newfd = open(&filename[2], - O_WRONLY | O_CREAT | O_APPEND, - 0666); - if (newfd < 0) { - if (!_terse) { - fprintf(stderr, - "ERROR: Could not open %s for append: %s\n", - &filename[2], - strerror(errno)); - fflush(stderr); - } - _exit(exitCodeOnFailure); - } - } else { - newfd = open(&filename[1], - O_WRONLY | O_CREAT | O_TRUNC, - 0666); - if (newfd < 0) { - if (!_terse) { - fprintf(stderr, - "ERROR: Could not open %s for write: %s\n", - &filename[1], - strerror(errno)); - fflush(stderr); - } - _exit(exitCodeOnFailure); - } - } - if (newfd != targetfd) { - dup2(newfd, targetfd); - CloseDescriptor(newfd); - } -} - - -bool -FastOS_UNIX_RealProcess:: -ForkAndExec(const char *command, - char **environmentVariables, - FastOS_UNIX_Process *process) -{ - bool rc = false; - int numArguments = 0; - char **execArgs = nullptr; - - if (!IsUsingShell()) { - numArguments = CountArguments(command); - if (numArguments > 0) { - execArgs = new char *[numArguments + 1]; - const char *nextArg = command; - - for(int i=0; ; i++) { - int length; - const char *arg = NextArgument(nextArg, &nextArg, &length); - - if (arg == nullptr) { - // printf("ARG nullptr\n"); - execArgs[i] = nullptr; - break; - } - // printf("argLen = %d\n", length); - execArgs[i] = new char[length + 1]; - memcpy(execArgs[i], arg, length); - execArgs[i][length] = '\0'; - // printf("arg %d: [%s]\n", i, execArgs[i]); - } - PrepareExecVPE(execArgs[0]); - } - } - - _pid = safe_fork(); - if (_pid == static_cast<pid_t>(0)) { - // Fork success, child side. - if (IsStdinPiped() && _stdinDes[0] != STDIN_FILENO) { - dup2(_stdinDes[0], STDIN_FILENO); - CloseDescriptor(_stdinDes[0]); - } - _stdinDes[0] = -1; - if (IsStdoutPiped() && _stdoutDes[1] != STDOUT_FILENO) { - dup2(_stdoutDes[1], STDOUT_FILENO); - CloseDescriptor(_stdoutDes[1]); - } - _stdoutDes[1] = -1; - if (IsStderrPiped() && _stderrDes[1] != STDERR_FILENO) { - dup2(_stderrDes[1], STDERR_FILENO); - CloseDescriptor(_stderrDes[1]); - } - _stderrDes[1] = -1; - // FIX! Check error codes for dup2, and do _exit(127) if trouble - - if ( ! _runDir.empty()) { - if (chdir(_runDir.c_str())) { - if (!_terse) { - fprintf(stderr, - "ERROR: Could not chdir to %s: %s\n", - _runDir.c_str(), - strerror(errno)); - fflush(stderr); - } - _exit(126); - } - } - RedirOut(_stdoutRedirName.c_str(), STDOUT_FILENO, 124); - RedirOut(_stderrRedirName.c_str(), STDERR_FILENO, 125); - - CloseDescriptor(_handshakeDes[0]); - _handshakeDes[0] = -1; - if (process != nullptr) { - int fdlimit = sysconf(_SC_OPEN_MAX); - // Close everything else - // printf("fdlimit = %d\n", fdlimit); - for(int fd = STDERR_FILENO + 1; fd < fdlimit; fd++) - { - if (fd != _handshakeDes[1]) - CloseDescriptor(fd); - } - } - if (fcntl(_handshakeDes[1], F_SETFD, FD_CLOEXEC) != 0) _exit(127); - - HandshakeWrite(0); - - // printf("exev(p)e [%s]\n", command); - if (IsUsingShell()) { - const char *shExecArgs[4]; - - shExecArgs[0] = "sh"; - shExecArgs[1] = "-c"; - shExecArgs[2] = command; - shExecArgs[3] = nullptr; - execve("/bin/sh", - const_cast<char *const *> - (reinterpret_cast<const char *const *> - (shExecArgs)), - environmentVariables); - int error = errno; - if (!_terse) { - fprintf(stderr, - "ERROR: Could not execv /bin/sh -c '%s': %s\n", - command, - strerror(error)); - fflush(stderr); - } - HandshakeWrite(error); - } - else - { - if (numArguments > 0) { - // printf("Command: [%s]\n", execArgs[0]); - ExecVPE(execArgs[0], - static_cast<char *const *>(execArgs), - environmentVariables); - int error = errno; - if (!_terse) { - fprintf(stderr, - "ERROR: Could not execve %s with " - "path search: %s\n", - execArgs[0], - strerror(error)); - fflush(stderr); - } - HandshakeWrite(error); - } - } - _exit(127); // If execve fails, we'll get it here - } - else if(_pid != static_cast<pid_t>(-1)) - { - /* Fork success, parent side */ - - // Close unused file descriptors - if (IsStdinPiped()) { - CloseAndResetDescriptor(&_stdinDes[0]); - } - if (IsStdoutPiped()) { - CloseAndResetDescriptor(&_stdoutDes[1]); - } - if (IsStderrPiped()) { - CloseAndResetDescriptor(&_stderrDes[1]); - } - - CloseAndResetDescriptor(&_handshakeDes[1]); - - int flags = fcntl(_handshakeDes[0], F_GETFL, 0); - if (flags != -1) { - flags &= ~O_NONBLOCK; - fcntl(_handshakeDes[0], F_SETFL, flags); - } - int phase1res = 0; - ssize_t rgot = HandshakeRead(&phase1res, sizeof(int)); - bool wasError = false; - int error = 0; - if (static_cast<size_t>(rgot) != sizeof(int)) wasError = true; - else if (phase1res != 0) { - wasError = true; - error = phase1res; - } else { - int phase2res = 0; - rgot = HandshakeRead(&phase2res, sizeof(int)); - if (rgot >= 1) { - if (static_cast<size_t>(rgot) >= sizeof(int)) - error = phase2res; - wasError = true; - } - } - - if (wasError) { - int status = 0; - CloseDescriptors(); - pid_t wpid = waitpid(_pid, &status, 0); - if (wpid <= 0) { - fprintf(stderr, "ERROR: Could not start process %s\n", command); - } else if (WIFEXITED(status)) { - status = WEXITSTATUS(status); - switch (status) { - case 124: - if ( ! _stdoutRedirName.empty() && - _stdoutRedirName[0] == '>') { - if (_stdoutRedirName[1] == '>') - fprintf(stderr, "ERROR: Could not open %s for append", &_stdoutRedirName[2]); - else - fprintf(stderr, "ERROR: Could not open %s for write", &_stdoutRedirName[1]); - } - break; - case 125: - if ( ! _stderrRedirName.empty() && - _stderrRedirName[0] == '>') { - if (_stderrRedirName[1] == '>') - fprintf(stderr, "ERROR: Could not open %s for append", &_stderrRedirName[2]); - else - fprintf(stderr, "ERROR: Could not open %s for write", &_stderrRedirName[1]); - } - break; - case 126: - if ( ! _runDir.empty()) { - fprintf(stderr, "ERROR: Could not chdir to %s\n", _runDir.c_str()); - } - break; - case 127: - if (error != 0) { - std::error_code ec(error, std::system_category()); - fprintf(stderr, "ERROR: Could not execve %s: %s\n", command, ec.message().c_str()); - } else - fprintf(stderr, "ERROR: Could not execve %s\n", command); - break; - default: - fprintf(stderr, "ERROR: Could not start process %s\n", command); - break; - } - } else { - fprintf(stderr, "ERROR: Could not start process %s\n", command); - } - fflush(stderr); - } else { - rc = true; - } - } - if (execArgs != nullptr) { - char **arg = execArgs; - while (*arg != nullptr) { - delete [] *arg; - arg++; - } - delete [] execArgs; - } - - return rc; -} - - -void -FastOS_UNIX_RealProcess::HandshakeWrite(int val) -{ - if (_handshakeDes[1] == -1) - return; - const void *wbuf = &val; - size_t residue = sizeof(val); - for (;;) { - /* - * XXX: Might need to use syscall(SYS_write....) to avoid - * thread library interference. - */ - ssize_t wgot = write(_handshakeDes[1], wbuf, residue); - if (wgot < 0 && errno == EINTR) - continue; - if (wgot <= 0) - break; - wbuf = static_cast<const char *>(wbuf) + wgot; - residue -= wgot; - if (residue == 0) - break; - } -} - - -ssize_t -FastOS_UNIX_RealProcess::HandshakeRead(void *buf, size_t len) -{ - if (_handshakeDes[0] == -1) - return 0; - size_t residue = len; - ssize_t rgot = 0; - void *rbuf = buf; - for (;;) { - rgot = read(_handshakeDes[0], rbuf, residue); - if (rgot < 0 && errno == EINTR) - continue; - if (rgot <= 0) - break; - rbuf = static_cast<char *>(rbuf) + rgot; - residue -= rgot; - if (residue == 0) - break; - } - return (residue == len) ? rgot : len - residue; -} - - -bool -FastOS_UNIX_RealProcess::Setup() -{ - bool rc = true; - - if (IsStdinPiped()) rc = rc && (pipe(_stdinDes) == 0); - if (IsStdoutPiped()) rc = rc && (pipe(_stdoutDes) == 0); - if (IsStderrPiped()) rc = rc && (pipe(_stderrDes) == 0); - rc = rc && (pipe(_handshakeDes) == 0); - return rc; -} - - -FastOS_UNIX_Process:: -FastOS_UNIX_Process (const char *cmdLine, - FastOS_ProcessRedirectListener *stdoutListener, - FastOS_ProcessRedirectListener *stderrListener) : - FastOS_ProcessInterface(cmdLine, stdoutListener, stderrListener), - _pid(0), - _died(false), - _returnCode(-1), - _descriptor(), - _runDir(), - _stdoutRedirName(), - _stderrRedirName(), - _killed(false), - _closing(nullptr) -{ - constexpr uint32_t RING_BUFFER_SIZE = 0x10000; - if (stdoutListener != nullptr) - _descriptor[TYPE_STDOUT]._readBuffer = std::make_unique<FastOS_RingBuffer>(RING_BUFFER_SIZE); - if (stderrListener != nullptr) - _descriptor[TYPE_STDERR]._readBuffer = std::make_unique<FastOS_RingBuffer>(RING_BUFFER_SIZE); - - { - auto guard = _app->getProcessGuard(); - _app->AddChildProcess(this); - } - - // App::AddToIPCComm() is performed when the process is started -} - -FastOS_UNIX_Process::~FastOS_UNIX_Process () -{ - Kill(); // Kill if not dead or detached. - - if ((GetDescriptorHandle(TYPE_STDOUT)._fd != -1) || - (GetDescriptorHandle(TYPE_STDERR)._fd != -1)) - { - // Let the IPC helper flush write queues and remove us from the - // process list before we disappear. - static_cast<FastOS_UNIX_Application *>(_app)->RemoveFromIPCComm(this); - } else { - // No IPC descriptor, do it ourselves - auto guard = _app->getProcessGuard(); - _app->RemoveChildProcess(this); - } - - for(int i=0; i<int(TYPE_COUNT); i++) { - _descriptor[i]._readBuffer.reset(); - _descriptor[i]._writeBuffer.reset(); - CloseDescriptor(DescriptorType(i)); - } - - CloseListener(TYPE_STDOUT); - CloseListener(TYPE_STDERR); -} - -bool FastOS_UNIX_Process::CreateInternal (bool useShell) -{ - return GetProcessStarter()->CreateProcess(this, useShell, - _stdoutListener != nullptr, - _stderrListener != nullptr); -} - -bool FastOS_UNIX_Process::WriteStdin (const void *data, size_t length) -{ - bool rc = false; - DescriptorHandle &desc = GetDescriptorHandle(TYPE_STDIN); - - if (desc._fd != -1) { - if (data == nullptr) { - CloseDescriptor(TYPE_STDIN); - rc = true; - } - else - { - int writerc = write(desc._fd, data, length); - if (writerc < int(length)) - CloseDescriptor(TYPE_STDIN); - else - rc = true; - } - } - - return rc; -} - -bool FastOS_UNIX_Process::Signal(int sig) -{ - bool rc = false; - pid_t pid; - - auto guard = _app->getProcessGuard(); - pid = GetProcessId(); - if (pid == 0) { - /* Do nothing */ - } else if (GetDeathFlag()) { - rc = true; // The process is no longer around. - } else if (kill(pid, sig) == 0) { - if (sig == SIGKILL) - _killed = true; - rc = true; - } - return rc; -} - -bool FastOS_UNIX_Process::Kill () -{ - return Signal(SIGKILL); -} - -bool FastOS_UNIX_Process::InternalWait (int *returnCode, - int timeOutSeconds, - bool *pollStillRunning) -{ - bool rc = GetProcessStarter()->Wait(this, timeOutSeconds, - pollStillRunning); - if (rc) { - if (_killed) - *returnCode = KILL_EXITCODE; - else - *returnCode = _returnCode; - } - - return rc; -} - -bool FastOS_UNIX_Process::Wait (int *returnCode, int timeOutSeconds) -{ - return InternalWait(returnCode, timeOutSeconds, nullptr); -} - -bool FastOS_UNIX_Process::PollWait (int *returnCode, bool *stillRunning) -{ - return InternalWait(returnCode, -1, stillRunning); -} - -int FastOS_UNIX_Process::BuildStreamMask (bool useShell) -{ - int streamMask = FastOS_UNIX_RealProcess::STREAM_STDIN; - if (_stdoutListener) streamMask |= FastOS_UNIX_RealProcess::STREAM_STDOUT; - if (_stderrListener) streamMask |= FastOS_UNIX_RealProcess::STREAM_STDERR; - if (useShell) streamMask |= FastOS_UNIX_RealProcess::EXEC_SHELL; - - return streamMask; -} - - -FastOS_UNIX_ProcessStarter::FastOS_UNIX_ProcessStarter (FastOS_ApplicationInterface *app) - : _app(app), - _hasDirectChildren(false) -{ -} - -FastOS_UNIX_ProcessStarter::~FastOS_UNIX_ProcessStarter () = default; - -bool -FastOS_UNIX_ProcessStarter:: -CreateProcess (FastOS_UNIX_Process *process, bool useShell, bool pipeStdout, bool pipeStderr) -{ - bool rc = false; - - const char *cmdLine = process->GetCommandLine(); - - auto guard = _app->getProcessGuard(); - - _hasDirectChildren = true; - FastOS_UNIX_RealProcess *rprocess = new FastOS_UNIX_RealProcess(process->BuildStreamMask(useShell)); - const char *runDir = process->GetRunDir(); - if (runDir != nullptr) { - rprocess->SetRunDir(runDir); // Handover - } - const char *stdoutRedirName = process->GetStdoutRedirName(); - if (stdoutRedirName != nullptr) { - rprocess->SetStdoutRedirName(stdoutRedirName); - } - const char *stderrRedirName = process->GetStderrRedirName(); - if (stderrRedirName != nullptr) { - rprocess->SetStderrRedirName(stderrRedirName); - } - rprocess->SetTerse(); - rprocess->Setup(); - process->SetDescriptor(FastOS_UNIX_Process::TYPE_STDIN, rprocess->HandoverStdinDescriptor()); - if (pipeStdout) - process->SetDescriptor(FastOS_UNIX_Process::TYPE_STDOUT, rprocess->HandoverStdoutDescriptor()); - if (pipeStderr) - process->SetDescriptor(FastOS_UNIX_Process::TYPE_STDERR, rprocess->HandoverStderrDescriptor()); - pid_t processId = -1; - if (rprocess->ForkAndExec(cmdLine, environ, process)) { - processId = rprocess->GetProcessID(); - } - if (processId != -1) { - process->SetProcessId(static_cast<unsigned int>(processId)); - if (!useShell || pipeStdout || pipeStderr) - static_cast<FastOS_UNIX_Application *>(_app)->AddToIPCComm(process); - rc = true; - } else { - fprintf(stderr, "Forkandexec %s failed\n", cmdLine); - } - guard.unlock(); - delete rprocess; - return rc; -} - - -void -FastOS_UNIX_ProcessStarter::PollReapDirectChildren() -{ - int status; - pid_t pid; - - for (;;) { - pid = waitpid(-1, &status, WNOHANG); - if (pid <= 0) - break; - - FastOS_ProcessInterface *node; - for(node = _app->GetProcessList(); - node != nullptr; node = node->_next) - { - FastOS_UNIX_Process *xproc = static_cast<FastOS_UNIX_Process *>(node); - - if (xproc->GetProcessId() == static_cast<unsigned int>(pid)) - xproc->DeathNotification(normalizedWaitStatus(status)); - } - } -} - - -bool -FastOS_UNIX_ProcessStarter::Wait(FastOS_UNIX_Process *process, - int timeOutSeconds, - bool *pollStillRunning) -{ - bool rc = true; - - bool timeOutKillAttempted = false; - - steady_clock::time_point start = steady_clock::now(); - - if (pollStillRunning != nullptr) - *pollStillRunning = true; - - for (;;) { - { - auto guard = process->_app->getProcessGuard(); - - if (_hasDirectChildren) PollReapDirectChildren(); - } - - if (process->GetDeathFlag()) { - if (pollStillRunning != nullptr) - *pollStillRunning = false; - break; - } - - if (pollStillRunning != nullptr) - break; - - if ((timeOutSeconds != -1) && !timeOutKillAttempted) { - - if ((steady_clock::now() - start) >= seconds(timeOutSeconds)) { - process->Kill(); - timeOutKillAttempted = true; - } - } - - std::this_thread::sleep_for(100ms); - } - - return rc; -} - -FastOS_UNIX_Process::DescriptorHandle::DescriptorHandle() - : _fd(-1), - _wantRead(false), - _wantWrite(false), - _canRead(false), - _canWrite(false), - _pollIdx(-1), - _readBuffer(), - _writeBuffer() -{ -} -FastOS_UNIX_Process::DescriptorHandle::~DescriptorHandle() = default; -void -FastOS_UNIX_Process::DescriptorHandle::CloseHandle() -{ - _wantRead = false; - _wantWrite = false; - _canRead = false; - _canWrite = false; - _pollIdx = -1; - if (_fd != -1) { - close(_fd); - _fd = -1; - } - if (_readBuffer) - _readBuffer->Close(); - if (_writeBuffer) - _writeBuffer->Close(); -} diff --git a/fastos/src/vespa/fastos/unix_process.h b/fastos/src/vespa/fastos/unix_process.h deleted file mode 100644 index 43b4388e0c3..00000000000 --- a/fastos/src/vespa/fastos/unix_process.h +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -//************************************************************************ -/** - * Class definitions for FastOS_UNIX_Process. - * - * @author Div, Oivind H. Danielsen - */ - -#pragma once - -#include "process.h" -#include "app.h" -#include <string> -#include <memory> -#include <future> - -class FastOS_UNIX_RealProcess; -class FastOS_RingBuffer; - -class FastOS_UNIX_Process : public FastOS_ProcessInterface -{ -private: - FastOS_UNIX_Process(const FastOS_UNIX_Process&); - FastOS_UNIX_Process& operator=(const FastOS_UNIX_Process&); - - unsigned int _pid; - bool _died; - int _returnCode; -public: - class DescriptorHandle - { - private: - DescriptorHandle(const DescriptorHandle &); - DescriptorHandle& operator=(const DescriptorHandle &); - - public: - int _fd; - bool _wantRead; - bool _wantWrite; - bool _canRead; - bool _canWrite; - int _pollIdx; - std::unique_ptr<FastOS_RingBuffer> _readBuffer; - std::unique_ptr<FastOS_RingBuffer> _writeBuffer; - DescriptorHandle(); - ~DescriptorHandle(); - void CloseHandle(); - }; -private: - DescriptorHandle _descriptor[4]; - - std::string _runDir; - std::string _stdoutRedirName; - std::string _stderrRedirName; - bool _killed; - - FastOS_UNIX_ProcessStarter *GetProcessStarter () { - return static_cast<FastOS_UNIX_Application *>(_app)->GetProcessStarter(); - } - - bool InternalWait (int *returnCode, int timeOutSeconds, bool *pollStillRunning); -public: - enum DescriptorType - { - TYPE_STDOUT, - TYPE_STDERR, - TYPE_STDIN, - TYPE_COUNT - }; - - enum Constants - { - TYPE_READCOUNT = 3 - }; - std::unique_ptr<std::promise<void>> _closing; - FastOS_ProcessRedirectListener *GetListener (DescriptorType type) - { - if(type == TYPE_STDOUT) - return _stdoutListener; - else if(type == TYPE_STDERR) - return _stderrListener; - - return nullptr; - } - - void CloseListener (DescriptorType type) - { - if(type == TYPE_STDOUT) - { - if(_stdoutListener != nullptr) - { - _stdoutListener->OnReceiveData(nullptr, 0); - _stdoutListener = nullptr; - } - } - else if(type == TYPE_STDERR) - { - if(_stderrListener != nullptr) - { - _stderrListener->OnReceiveData(nullptr, 0); - _stderrListener = nullptr; - } - } - } - - FastOS_UNIX_Process (const char *cmdLine, - FastOS_ProcessRedirectListener *stdoutListener = nullptr, - FastOS_ProcessRedirectListener *stderrListener = nullptr); - ~FastOS_UNIX_Process (); - bool CreateInternal (bool useShell); - bool Create () override { return CreateInternal(false); } - bool CreateWithShell () override { return CreateInternal(true); } - bool WriteStdin (const void *data, size_t length) override; - bool Signal(int sig); - bool Kill () override; - bool Wait (int *returnCode, int timeOutSeconds = -1) override; - bool PollWait (int *returnCode, bool *stillRunning) override; - void SetProcessId (unsigned int pid) { _pid = pid; } - unsigned int GetProcessId() override { return _pid; } - void DeathNotification (int returnCode) { - _returnCode = returnCode; - _died = true; - } - bool GetDeathFlag () { return _died; } - int BuildStreamMask (bool useShell); - - void CloseDescriptor (DescriptorType type) - { - _descriptor[type].CloseHandle(); - } - - void SetDescriptor (DescriptorType type, int descriptor) - { - _descriptor[type]._fd = descriptor; - } - - DescriptorHandle &GetDescriptorHandle(DescriptorType type) - { - return _descriptor[type]; - } - - bool GetKillFlag () { return _killed; } - - const char *GetRunDir() const { return _runDir.c_str(); } - const char *GetStdoutRedirName() const { return _stdoutRedirName.c_str(); } - const char *GetStderrRedirName() const { return _stderrRedirName.c_str(); } -}; - - -class FastOS_UNIX_RealProcess; -class FastOS_UNIX_ProcessStarter -{ -private: - FastOS_UNIX_ProcessStarter(const FastOS_UNIX_ProcessStarter&); - FastOS_UNIX_ProcessStarter& operator=(const FastOS_UNIX_ProcessStarter&); - -protected: - FastOS_ApplicationInterface *_app; - bool _hasDirectChildren; - - void PollReapDirectChildren(); - -public: - FastOS_UNIX_ProcessStarter (FastOS_ApplicationInterface *app); - ~FastOS_UNIX_ProcessStarter (); - - bool CreateProcess (FastOS_UNIX_Process *process, bool useShell, - bool pipeStdout, bool pipeStderr); - bool Wait (FastOS_UNIX_Process *process, int timeOutSeconds, bool *pollStillRunning); -}; - - diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 8960fed5ef0..4f889968897 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -216,7 +216,7 @@ public class Flags { ); public static final UnboundBooleanFlag DELETE_UNMAINTAINED_CERTIFICATES = defineFeatureFlag( - "delete-unmaintained-certificates", false, + "delete-unmaintained-certificates", true, List.of("andreer"), "2021-09-23", "2022-03-14", "Whether to delete certificates that are known by provider but not by controller", "Takes effect on next run of EndpointCertificateMaintainer" @@ -397,6 +397,13 @@ public class Flags { "Takes effect at redeployment", APPLICATION_ID); + public static final UnboundBooleanFlag EXPERIMENTAL_SD_PARSING = defineFeatureFlag( + "experimental-sd-parsing", false, + List.of("arnej"), "2022-03-04", "2022-12-31", + "Parsed schema files via intermediate format", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/fnet/src/tests/examples/examples_test.cpp b/fnet/src/tests/examples/examples_test.cpp index dc7b051bd80..b6476311adb 100644 --- a/fnet/src/tests/examples/examples_test.cpp +++ b/fnet/src/tests/examples/examples_test.cpp @@ -1,6 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/child_process.h> +#include <vespa/vespalib/process/process.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/thread.h> @@ -11,209 +11,200 @@ static const int PORT0 = 18570; static const int PORT1 = 18571; -using vespalib::ChildProcess; +using vespalib::Process; +using vespalib::make_string_short::fmt; -bool runProc(ChildProcess &proc, std::atomic<bool> &done) { - char buf[4_Ki]; - proc.close(); // close stdin - while (proc.running() && !done) { - if (!proc.eof()) { - uint32_t res = proc.read(buf, sizeof(buf), 10); - std::string tmp(buf, res); - fprintf(stderr, "%s", tmp.c_str()); - } - vespalib::Thread::sleep(10); +int run_proc(Process &proc, vespalib::string &output) { + proc.close(); + for (auto mem = proc.obtain(); mem.size > 0; mem = proc.obtain()) { + output.append(mem.data, mem.size); + proc.evict(mem.size); } - if (done && proc.running()) { - kill(proc.getPid(), SIGTERM); - return proc.wait(60000); + return proc.join(); +} + +void consume_result(Process &proc) { + vespalib::string output; + int status = run_proc(proc, output); + fprintf(stderr, "child output(server): >>>%s<<<\n", output.c_str()); + if (status != 0) { + // Allow 'killed by SIGTERM' result status. This is needed as + // some clients will exit with success status even when the + // server is not yet running, resulting in the server being + // killed before it has installed any signal handlers. + EXPECT_TRUE(status & 0x80000000); + status &= 0x7fffffff; + EXPECT_TRUE(WIFSIGNALED(status)); + EXPECT_EQUAL(WTERMSIG(status), SIGTERM); } - return !proc.failed(); } -bool runProc(const std::string &cmd) { - bool ok = false; - for (size_t retry = 0; !ok && retry < 60; ++retry) { +bool run_with_retry(const vespalib::string &cmd) { + for (size_t retry = 0; retry < 60; ++retry) { if (retry > 0) { fprintf(stderr, "retrying command in 500ms...\n"); vespalib::Thread::sleep(500); } - std::atomic<bool> done(false); - ChildProcess proc(cmd.c_str()); - ok = runProc(proc, done); + vespalib::string output; + Process proc(cmd, true); + int status = run_proc(proc, output); + fprintf(stderr, "child output(client): >>>%s<<<\n", output.c_str()); + if (status == 0) { + return true; + } } - return ok; + fprintf(stderr, "giving up...\n"); + return false; } TEST("usage") { - std::atomic<bool> done(false); - { - ChildProcess proc("exec ../../examples/proxy/fnet_proxy_app"); - EXPECT_FALSE(runProc(proc, done)); - } - { - ChildProcess proc("exec ../../examples/ping/fnet_pingserver_app"); - EXPECT_FALSE(runProc(proc, done)); - } - { - ChildProcess proc("exec ../../examples/ping/fnet_pingclient_app"); - EXPECT_FALSE(runProc(proc, done)); - } - { - ChildProcess proc("exec ../../examples/frt/rpc/fnet_rpc_client_app"); - EXPECT_FALSE(runProc(proc, done)); - } - { - ChildProcess proc("exec ../../examples/frt/rpc/fnet_rpc_server_app"); - EXPECT_FALSE(runProc(proc, done)); - } - { - ChildProcess proc("exec ../../examples/frt/rpc/fnet_echo_client_app"); - EXPECT_FALSE(runProc(proc, done)); - } - { - ChildProcess proc("exec ../../examples/frt/rpc/vespa-rpc-info"); - EXPECT_FALSE(runProc(proc, done)); - } - { - ChildProcess proc("exec ../../examples/frt/rpc/vespa-rpc-invoke-bin"); - EXPECT_FALSE(runProc(proc, done)); - } - { - ChildProcess proc("exec ../../examples/frt/rpc/fnet_rpc_callback_server_app"); - EXPECT_FALSE(runProc(proc, done)); - } - { - ChildProcess proc("exec ../../examples/frt/rpc/fnet_rpc_callback_client_app"); - EXPECT_FALSE(runProc(proc, done)); - } + EXPECT_FALSE(Process::run("exec ../../examples/proxy/fnet_proxy_app")); + EXPECT_FALSE(Process::run("exec ../../examples/ping/fnet_pingserver_app")); + EXPECT_FALSE(Process::run("exec ../../examples/ping/fnet_pingclient_app")); + EXPECT_FALSE(Process::run("exec ../../examples/frt/rpc/fnet_rpc_client_app")); + EXPECT_FALSE(Process::run("exec ../../examples/frt/rpc/fnet_rpc_server_app")); + EXPECT_FALSE(Process::run("exec ../../examples/frt/rpc/fnet_echo_client_app")); + EXPECT_FALSE(Process::run("exec ../../examples/frt/rpc/vespa-rpc-info")); + EXPECT_FALSE(Process::run("exec ../../examples/frt/rpc/vespa-rpc-invoke-bin")); + EXPECT_FALSE(Process::run("exec ../../examples/frt/rpc/fnet_rpc_callback_server_app")); + EXPECT_FALSE(Process::run("exec ../../examples/frt/rpc/fnet_rpc_callback_client_app")); } TEST("timeout") { - std::string out; - EXPECT_TRUE(ChildProcess::run("exec ../../examples/timeout/fnet_timeout_app", out)); + vespalib::string out; + EXPECT_TRUE(Process::run("exec ../../examples/timeout/fnet_timeout_app", out)); fprintf(stderr, "%s\n", out.c_str()); } -TEST_MT_F("ping", 2, std::atomic<bool>()) { +TEST_MT_F("ping", 2, pid_t(-1)) { if (thread_id == 0) { - ChildProcess proc(vespalib::make_string("exec ../../examples/ping/fnet_pingserver_app tcp/%d", - PORT0).c_str()); + Process proc(fmt("exec ../../examples/ping/fnet_pingserver_app tcp/%d", + PORT0), true); + f1 = proc.pid(); TEST_BARRIER(); - EXPECT_TRUE(runProc(proc, f1)); + TEST_DO(consume_result(proc)); } else { TEST_BARRIER(); - EXPECT_TRUE(runProc(vespalib::make_string("exec ../../examples/ping/fnet_pingclient_app tcp/localhost:%d", - PORT0).c_str())); - f1 = true; + EXPECT_TRUE(run_with_retry(fmt("exec ../../examples/ping/fnet_pingclient_app tcp/localhost:%d", + PORT0))); + kill(f1, SIGTERM); } } -TEST_MT_F("ping times out", 2, std::atomic<bool>()) { +TEST_MT_F("ping times out", 2, pid_t(-1)) { if (thread_id == 0) { - ChildProcess proc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_rpc_server_app tcp/%d", - PORT0).c_str()); + Process proc(fmt("exec ../../examples/frt/rpc/fnet_rpc_server_app tcp/%d", + PORT0), true); + f1 = proc.pid(); TEST_BARRIER(); - EXPECT_TRUE(runProc(proc, f1)); + TEST_DO(consume_result(proc)); } else { float timeout_s = 0.1; TEST_BARRIER(); - EXPECT_TRUE(runProc(vespalib::make_string("exec ../../examples/ping/fnet_pingclient_app tcp/localhost:%d %f", - PORT0, timeout_s).c_str())); - f1 = true; + EXPECT_TRUE(run_with_retry(fmt("exec ../../examples/ping/fnet_pingclient_app tcp/localhost:%d %f", + PORT0, timeout_s))); + kill(f1, SIGTERM); } } -TEST_MT_F("ping with proxy", 3, std::atomic<bool>()) { +TEST_MT_FF("ping with proxy", 3, pid_t(-1), pid_t(-1)) { if (thread_id == 0) { - ChildProcess proc(vespalib::make_string("exec ../../examples/ping/fnet_pingserver_app tcp/%d", - PORT0).c_str()); + Process proc(fmt("exec ../../examples/ping/fnet_pingserver_app tcp/%d", + PORT0), true); + f1 = proc.pid(); TEST_BARRIER(); - EXPECT_TRUE(runProc(proc, f1)); + TEST_DO(consume_result(proc)); } else if (thread_id == 1) { - ChildProcess proc(vespalib::make_string("exec ../../examples/proxy/fnet_proxy_app tcp/%d tcp/localhost:%d", - PORT1, PORT0).c_str()); + Process proc(fmt("exec ../../examples/proxy/fnet_proxy_app tcp/%d tcp/localhost:%d", + PORT1, PORT0), true); + f2 = proc.pid(); TEST_BARRIER(); - EXPECT_TRUE(runProc(proc, f1)); + TEST_DO(consume_result(proc)); } else { TEST_BARRIER(); - EXPECT_TRUE(runProc(vespalib::make_string("exec ../../examples/ping/fnet_pingclient_app tcp/localhost:%d", - PORT1).c_str())); - f1 = true; + EXPECT_TRUE(run_with_retry(fmt("exec ../../examples/ping/fnet_pingclient_app tcp/localhost:%d", + PORT1))); + kill(f1, SIGTERM); + kill(f2, SIGTERM); } } -TEST_MT_F("rpc client server", 2, std::atomic<bool>()) { +TEST_MT_F("rpc client server", 2, pid_t(-1)) { if (thread_id == 0) { - ChildProcess proc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_rpc_server_app tcp/%d", - PORT0).c_str()); + Process proc(fmt("exec ../../examples/frt/rpc/fnet_rpc_server_app tcp/%d", + PORT0), true); + f1 = proc.pid(); TEST_BARRIER(); - EXPECT_TRUE(runProc(proc, f1)); + TEST_DO(consume_result(proc)); } else { TEST_BARRIER(); - EXPECT_TRUE(runProc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_rpc_client_app tcp/localhost:%d", - PORT0).c_str())); - f1 = true; + EXPECT_TRUE(run_with_retry(fmt("exec ../../examples/frt/rpc/fnet_rpc_client_app tcp/localhost:%d", + PORT0))); + kill(f1, SIGTERM); } } -TEST_MT_F("rpc echo client", 2, std::atomic<bool>()) { +TEST_MT_F("rpc echo client", 2, pid_t(-1)) { if (thread_id == 0) { - ChildProcess proc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_rpc_server_app tcp/%d", - PORT0).c_str()); + Process proc(fmt("exec ../../examples/frt/rpc/fnet_rpc_server_app tcp/%d", + PORT0), true); + f1 = proc.pid(); TEST_BARRIER(); - EXPECT_TRUE(runProc(proc, f1)); + TEST_DO(consume_result(proc)); } else { TEST_BARRIER(); - EXPECT_TRUE(runProc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_echo_client_app tcp/localhost:%d", - PORT0).c_str())); - f1 = true; + EXPECT_TRUE(run_with_retry(fmt("exec ../../examples/frt/rpc/fnet_echo_client_app tcp/localhost:%d", + PORT0))); + kill(f1, SIGTERM); } } -TEST_MT_F("rpc info", 2, std::atomic<bool>()) { +TEST_MT_F("rpc info", 2, pid_t(-1)) { if (thread_id == 0) { - ChildProcess proc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_rpc_server_app tcp/%d", - PORT0).c_str()); + Process proc(fmt("exec ../../examples/frt/rpc/fnet_rpc_server_app tcp/%d", + PORT0), true); + f1 = proc.pid(); TEST_BARRIER(); - EXPECT_TRUE(runProc(proc, f1)); + TEST_DO(consume_result(proc)); } else { TEST_BARRIER(); - EXPECT_TRUE(runProc(vespalib::make_string("exec ../../examples/frt/rpc/vespa-rpc-info tcp/localhost:%d", - PORT0).c_str())); - EXPECT_TRUE(runProc(vespalib::make_string("exec ../../examples/frt/rpc/vespa-rpc-info tcp/localhost:%d verbose", - PORT0).c_str())); - f1 = true; + EXPECT_TRUE(run_with_retry(fmt("exec ../../examples/frt/rpc/vespa-rpc-info tcp/localhost:%d", + PORT0))); + EXPECT_TRUE(run_with_retry(fmt("exec ../../examples/frt/rpc/vespa-rpc-info tcp/localhost:%d verbose", + PORT0))); + kill(f1, SIGTERM); } } -TEST_MT_F("rpc invoke", 2, std::atomic<bool>()) { +TEST_MT_F("rpc invoke", 2, pid_t(-1)) { if (thread_id == 0) { - ChildProcess proc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_rpc_server_app tcp/%d", - PORT0).c_str()); + Process proc(fmt("exec ../../examples/frt/rpc/fnet_rpc_server_app tcp/%d", + PORT0), true); + f1 = proc.pid(); TEST_BARRIER(); - EXPECT_TRUE(runProc(proc, f1)); + TEST_DO(consume_result(proc)); } else { TEST_BARRIER(); - EXPECT_TRUE(runProc(vespalib::make_string("exec ../../examples/frt/rpc/vespa-rpc-invoke-bin tcp/localhost:%d frt.rpc.echo " - "b:1 h:2 i:4 l:8 f:0.5 d:0.25 s:foo", - PORT0).c_str())); - f1 = true; + EXPECT_TRUE(run_with_retry(fmt("exec ../../examples/frt/rpc/vespa-rpc-invoke-bin tcp/localhost:%d frt.rpc.echo " + "b:1 h:2 i:4 l:8 f:0.5 d:0.25 s:foo", + PORT0))); + kill(f1, SIGTERM); } } -TEST_MT_F("rpc callback client server", 2, std::atomic<bool>()) { +TEST_MT_F("rpc callback client server", 2, pid_t(-1)) { if (thread_id == 0) { - ChildProcess proc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_rpc_callback_server_app tcp/%d", - PORT0).c_str()); + Process proc(fmt("exec ../../examples/frt/rpc/fnet_rpc_callback_server_app tcp/%d", + PORT0), true); + f1 = proc.pid(); TEST_BARRIER(); - EXPECT_TRUE(runProc(proc, f1)); + TEST_DO(consume_result(proc)); } else { TEST_BARRIER(); - EXPECT_TRUE(runProc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_rpc_callback_client_app tcp/localhost:%d", - PORT0).c_str())); - f1 = true; + EXPECT_TRUE(run_with_retry(fmt("exec ../../examples/frt/rpc/fnet_rpc_callback_client_app tcp/localhost:%d", + PORT0))); + kill(f1, SIGTERM); } } -TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.h b/persistence/src/vespa/persistence/spi/persistenceprovider.h index 8c62e691daf..38c86454255 100644 --- a/persistence/src/vespa/persistence/spi/persistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/persistenceprovider.h @@ -8,6 +8,7 @@ #include "selection.h" #include "clusterstate.h" #include "operationcomplete.h" +#include <vespa/document/base/documentid.h> namespace document { class FieldSet; } namespace vespalib { class IDestructorCallback; } diff --git a/searchcommon/src/vespa/searchcommon/attribute/status.cpp b/searchcommon/src/vespa/searchcommon/attribute/status.cpp index 0509e9a51c4..a7d1f5b3d38 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/status.cpp +++ b/searchcommon/src/vespa/searchcommon/attribute/status.cpp @@ -1,6 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "status.h" +#include <vespa/vespalib/util/atomic.h> + +using namespace vespalib::atomic; + namespace search::attribute { Status::Status() @@ -21,15 +25,15 @@ Status::Status() } Status::Status(const Status& rhs) - : _numDocs(rhs._numDocs), - _numValues(rhs._numValues), - _numUniqueValues(rhs._numUniqueValues), - _allocated(rhs._allocated), - _used(rhs._used), - _dead(rhs._dead), - _unused(rhs._unused), - _onHold(rhs._onHold), - _onHoldMax(rhs._onHoldMax), + : _numDocs(load_relaxed(rhs._numDocs)), + _numValues(load_relaxed(rhs._numValues)), + _numUniqueValues(load_relaxed(rhs._numUniqueValues)), + _allocated(load_relaxed(rhs._allocated)), + _used(load_relaxed(rhs._used)), + _dead(load_relaxed(rhs._dead)), + _unused(load_relaxed(rhs._unused)), + _onHold(load_relaxed(rhs._onHold)), + _onHoldMax(load_relaxed(rhs._onHoldMax)), _lastSyncToken(rhs.getLastSyncToken()), _updates(rhs._updates), _nonIdempotentUpdates(rhs._nonIdempotentUpdates), @@ -40,15 +44,15 @@ Status::Status(const Status& rhs) Status& Status::operator=(const Status& rhs) { - _numDocs = rhs._numDocs; - _numValues = rhs._numValues; - _numUniqueValues = rhs._numUniqueValues; - _allocated = rhs._allocated; - _used = rhs._used; - _dead = rhs._dead; - _unused = rhs._unused; - _onHold = rhs._onHold; - _onHoldMax = rhs._onHoldMax; + store_relaxed(_numDocs, load_relaxed(rhs._numDocs)); + store_relaxed(_numValues, load_relaxed(rhs._numValues)); + store_relaxed(_numUniqueValues, load_relaxed(rhs._numUniqueValues)); + store_relaxed(_allocated, load_relaxed(rhs._allocated)); + store_relaxed(_used, load_relaxed(rhs._used)); + store_relaxed(_dead, load_relaxed(rhs._dead)); + store_relaxed(_unused, load_relaxed(rhs._unused)); + store_relaxed(_onHold, load_relaxed(rhs._onHold)); + store_relaxed(_onHoldMax, load_relaxed(rhs._onHoldMax)); setLastSyncToken(rhs.getLastSyncToken()); _updates = rhs._updates; _nonIdempotentUpdates = rhs._nonIdempotentUpdates; @@ -69,14 +73,14 @@ void Status::updateStatistics(uint64_t numValues, uint64_t numUniqueValue, uint64_t allocated, uint64_t used, uint64_t dead, uint64_t onHold) { - _numValues = numValues; - _numUniqueValues = numUniqueValue; - _allocated = allocated; - _used = used; - _dead = dead; - _unused = allocated - used; - _onHold = onHold; - _onHoldMax = std::max(_onHoldMax, onHold); + store_relaxed(_numValues, numValues); + store_relaxed(_numUniqueValues, numUniqueValue); + store_relaxed(_allocated, allocated); + store_relaxed(_used, used); + store_relaxed(_dead, dead); + store_relaxed(_unused, allocated - used); + store_relaxed(_onHold, onHold); + store_relaxed(_onHoldMax, std::max(load_relaxed(_onHoldMax), onHold)); } } diff --git a/searchcommon/src/vespa/searchcommon/attribute/status.h b/searchcommon/src/vespa/searchcommon/attribute/status.h index 0280bfaae4e..f2212d4c76a 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/status.h +++ b/searchcommon/src/vespa/searchcommon/attribute/status.h @@ -17,22 +17,23 @@ public: void updateStatistics(uint64_t numValues, uint64_t numUniqueValue, uint64_t allocated, uint64_t used, uint64_t dead, uint64_t onHold); - uint64_t getNumDocs() const { return _numDocs; } - uint64_t getNumValues() const { return _numValues; } - uint64_t getNumUniqueValues() const { return _numUniqueValues; } - uint64_t getAllocated() const { return _allocated; } - uint64_t getUsed() const { return _used; } - uint64_t getDead() const { return _dead; } - uint64_t getOnHold() const { return _onHold; } - uint64_t getOnHoldMax() const { return _onHoldMax; } + uint64_t getNumDocs() const { return _numDocs.load(std::memory_order_relaxed); } + uint64_t getNumValues() const { return _numValues.load(std::memory_order_relaxed); } + uint64_t getNumUniqueValues() const { return _numUniqueValues.load(std::memory_order_relaxed); } + uint64_t getAllocated() const { return _allocated.load(std::memory_order_relaxed); } + uint64_t getUsed() const { return _used.load(std::memory_order_relaxed); } + uint64_t getDead() const { return _dead.load(std::memory_order_relaxed); } + uint64_t getOnHold() const { return _onHold.load(std::memory_order_relaxed); } + uint64_t getOnHoldMax() const { return _onHoldMax.load(std::memory_order_relaxed); } // This might be accessed from other threads than the writer thread. uint64_t getLastSyncToken() const { return _lastSyncToken.load(std::memory_order_relaxed); } uint64_t getUpdateCount() const { return _updates; } uint64_t getNonIdempotentUpdateCount() const { return _nonIdempotentUpdates; } uint32_t getBitVectors() const { return _bitVectors; } - void setNumDocs(uint64_t v) { _numDocs = v; } - void incNumDocs() { ++_numDocs; } + void setNumDocs(uint64_t v) { _numDocs.store(v, std::memory_order_relaxed); } + void incNumDocs() { _numDocs.store(_numDocs.load(std::memory_order_relaxed) + 1u, + std::memory_order_relaxed); } void setLastSyncToken(uint64_t v) { _lastSyncToken.store(v, std::memory_order_relaxed); } void incUpdates(uint64_t v=1) { _updates += v; } void incNonIdempotentUpdates(uint64_t v = 1) { _nonIdempotentUpdates += v; } @@ -42,15 +43,15 @@ public: static vespalib::string createName(vespalib::stringref index, vespalib::stringref attr); private: - uint64_t _numDocs; - uint64_t _numValues; - uint64_t _numUniqueValues; - uint64_t _allocated; - uint64_t _used; - uint64_t _dead; - uint64_t _unused; - uint64_t _onHold; - uint64_t _onHoldMax; + std::atomic<uint64_t> _numDocs; + std::atomic<uint64_t> _numValues; + std::atomic<uint64_t> _numUniqueValues; + std::atomic<uint64_t> _allocated; + std::atomic<uint64_t> _used; + std::atomic<uint64_t> _dead; + std::atomic<uint64_t> _unused; + std::atomic<uint64_t> _onHold; + std::atomic<uint64_t> _onHoldMax; std::atomic<uint64_t> _lastSyncToken; uint64_t _updates; uint64_t _nonIdempotentUpdates; diff --git a/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp b/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp index ddb41ae67a6..85c39226998 100644 --- a/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp +++ b/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp @@ -3,19 +3,19 @@ #include <vespa/config/print/fileconfigwriter.h> #include <vespa/document/config/config-documenttypes.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/documentapi/documentapi.h> #include <vespa/messagebus/destinationsession.h> #include <vespa/messagebus/rpcmessagebus.h> #include <vespa/messagebus/network/rpcnetworkparams.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/signalhandler.h> -#include <vespa/vespalib/util/child_process.h> +#include <vespa/vespalib/process/process.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/config/common/exceptions.h> #include <vespa/config/helper/configgetter.hpp> - -#include <iostream> +#include <vespa/fastos/app.h> typedef vespalib::SignalHandler SIG; @@ -114,7 +114,6 @@ FeedHandler::~FeedHandler() class App : public FastOS_Application { public: - virtual bool useProcessStarter() const override { return true; } virtual int Main() override; }; @@ -200,8 +199,8 @@ App::Main() std::string feedCmd(vespalib::make_string("vespa-feeder --route \"%s\" %s", route.c_str(), feedFile.c_str())); fprintf(stderr, "running feed command: %s\n", feedCmd.c_str()); - std::string feederOutput; - bool feedingOk = vespalib::ChildProcess::run(feedCmd.c_str(), feederOutput); + vespalib::string feederOutput; + bool feedingOk = vespalib::Process::run(feedCmd, feederOutput); if (!feedingOk) { fprintf(stderr, "error: feed command failed\n"); fprintf(stderr, "feed command output:\n-----\n%s\n-----\n", feederOutput.c_str()); diff --git a/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp b/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp index 65d17d207ec..903b4d89696 100644 --- a/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp +++ b/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp @@ -50,7 +50,7 @@ makeIntArray(const std::vector<int32_t> &array) { auto result = std::make_unique<ArrayFieldValue>(arrayTypeInt); for (const auto &elem : array) { - result->append(std::make_unique<IntFieldValue>(elem)); + result->append(IntFieldValue::make(elem)); } return result; } @@ -60,7 +60,7 @@ makeStringArray(const std::vector<vespalib::string> &array) { auto result = std::make_unique<ArrayFieldValue>(arrayTypeString); for (const auto &elem : array) { - result->append(std::make_unique<StringFieldValue>(elem)); + result->append(StringFieldValue::make(elem)); } return result; } @@ -160,8 +160,8 @@ TEST_F("require that simple fields give simple values", SimpleFixture) auto doc = f.makeDoc(); doc->setValue(f.weightField, IntFieldValue(200)); doc->setValue(f.nameField, StringFieldValue("name200b")); - TEST_DO(f.assertExtracted("weight", std::make_unique<IntFieldValue>(200))); - TEST_DO(f.assertExtracted("name", std::make_unique<StringFieldValue>("name200b"))); + TEST_DO(f.assertExtracted("weight", IntFieldValue::make(200))); + TEST_DO(f.assertExtracted("name", StringFieldValue::make("name200b"))); } struct ArrayFixture : public FixtureBase diff --git a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp index 82eac88a53e..a460c65fbb2 100644 --- a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp +++ b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp @@ -338,8 +338,8 @@ const DocumentType &getAttrDocType() { IDocumentRetriever::SP doc_with_fields(const std::string &id, Timestamp t, Bucket b) { auto d = std::make_unique<Document>(getDocType(), DocumentId(id)); - d->set("header", "foo"); - d->set("body", "bar"); + d->setValue("header", StringFieldValue::make("foo")); + d->setValue("body", StringFieldValue::make("bar")); return std::make_shared<UnitDR>(getDocType(), std::move(d), t, b, false); } @@ -355,12 +355,12 @@ IDocumentRetriever::SP doc_with_attr_fields(const vespalib::string &id, const vespalib::string &attr_ss) { auto d = std::make_unique<Document>(getAttrDocType(), DocumentId(id)); - d->set("header", "foo"); - d->set("body", "bar"); - d->set("aa", aa); - d->set("ab", ab); - d->set("dd", dd); - d->set("ss", ss); + d->setValue("header", StringFieldValue::make("foo")); + d->setValue("body", StringFieldValue::make("bar")); + d->setValue("aa", IntFieldValue::make(aa)); + d->setValue("ab", IntFieldValue::make(ab)); + d->setValue("dd", DoubleFieldValue::make(dd)); + d->setValue("ss", StringFieldValue::make(ss)); return std::make_shared<AttrUnitDR>(std::move(d), t, b, false, attr_aa, attr_dd, attr_ss); } @@ -782,7 +782,7 @@ TEST("require that fieldset limits fields returned") { EXPECT_TRUE(res.isCompleted()); EXPECT_EQUAL(1u, res.getEntries().size()); Document expected(getDocType(), DocumentId("id:ns:foo::xxx1")); - expected.set("header", "foo"); + expected.setValue("header", StringFieldValue::make("foo")); TEST_DO(checkEntry(res, 0, expected, Timestamp(1))); } @@ -838,19 +838,19 @@ TEST("require that attributes are used") EXPECT_TRUE(res.isCompleted()); EXPECT_EQUAL(2u, res.getEntries().size()); Document expected1(getAttrDocType(), DocumentId("id:ns:foo::xx2")); - expected1.set("header", "foo"); - expected1.set("body", "bar"); - expected1.set("aa", 27); - expected1.set("ab", 28); - expected1.set("dd", 2.7); - expected1.set("ss", "x27"); + expected1.setValue("header", StringFieldValue::make("foo")); + expected1.setValue("body", StringFieldValue::make("bar")); + expected1.setValue("aa", IntFieldValue::make(27)); + expected1.setValue("ab", IntFieldValue::make(28)); + expected1.setValue("dd", DoubleFieldValue::make(2.7)); + expected1.setValue("ss", StringFieldValue::make("x27")); Document expected2(getAttrDocType(), DocumentId("id:ns:foo::xx4")); - expected2.set("header", "foo"); - expected2.set("body", "bar"); - expected2.set("aa", 45); - expected2.set("ab", 46); - expected2.set("dd", 4.5); - expected2.set("ss", "x45"); + expected2.setValue("header", StringFieldValue::make("foo")); + expected2.setValue("body", StringFieldValue::make("bar")); + expected2.setValue("aa", IntFieldValue::make(45)); + expected2.setValue("ab", IntFieldValue::make(46)); + expected2.setValue("dd", DoubleFieldValue::make(4.5)); + expected2.setValue("ss", StringFieldValue::make("x45")); TEST_DO(checkEntry(res, 0, expected1, Timestamp(2))); TEST_DO(checkEntry(res, 1, expected2, Timestamp(4))); @@ -868,19 +868,19 @@ TEST("require that attributes are used") EXPECT_TRUE(res2.isCompleted()); EXPECT_EQUAL(2u, res2.getEntries().size()); Document expected3(getAttrDocType(), DocumentId("id:ns:foo::xx6")); - expected3.set("header", "foo"); - expected3.set("body", "bar"); - expected3.set("aa", 27); - expected3.set("ab", 28); - expected3.set("dd", 2.7); - expected3.set("ss", "x27"); + expected3.setValue("header", StringFieldValue::make("foo")); + expected3.setValue("body", StringFieldValue::make("bar")); + expected3.setValue("aa", IntFieldValue::make(27)); + expected3.setValue("ab", IntFieldValue::make(28)); + expected3.setValue("dd", DoubleFieldValue::make(2.7)); + expected3.setValue("ss", StringFieldValue::make("x27")); Document expected4(getAttrDocType(), DocumentId("id:ns:foo::xx8")); - expected4.set("header", "foo"); - expected4.set("body", "bar"); - expected4.set("aa", 45); - expected4.set("ab", 46); - expected4.set("dd", 4.5); - expected4.set("ss", "x45"); + expected4.setValue("header", StringFieldValue::make("foo")); + expected4.setValue("body", StringFieldValue::make("bar")); + expected4.setValue("aa", IntFieldValue::make(45)); + expected4.setValue("ab", IntFieldValue::make(46)); + expected4.setValue("dd", DoubleFieldValue::make(4.5)); + expected4.setValue("ss", StringFieldValue::make("x45")); TEST_DO(checkEntry(res2, 0, expected3, Timestamp(6))); TEST_DO(checkEntry(res2, 1, expected4, Timestamp(8))); @@ -898,19 +898,19 @@ TEST("require that attributes are used") EXPECT_TRUE(res3.isCompleted()); EXPECT_EQUAL(2u, res3.getEntries().size()); Document expected5(getAttrDocType(), DocumentId("id:ns:foo::xx10")); - expected5.set("header", "foo"); - expected5.set("body", "bar"); - expected5.set("aa", 27); - expected5.set("ab", 28); - expected5.set("dd", 2.7); - expected5.set("ss", "x27"); + expected5.setValue("header", StringFieldValue::make("foo")); + expected5.setValue("body", StringFieldValue::make("bar")); + expected5.setValue("aa", IntFieldValue::make(27)); + expected5.setValue("ab", IntFieldValue::make(28)); + expected5.setValue("dd", DoubleFieldValue::make(2.7)); + expected5.setValue("ss", StringFieldValue::make("x27")); Document expected6(getAttrDocType(), DocumentId("id:ns:foo::xx12")); - expected6.set("header", "foo"); - expected6.set("body", "bar"); - expected6.set("aa", 45); - expected6.set("ab", 46); - expected6.set("dd", 4.5); - expected6.set("ss", "x45"); + expected6.setValue("header", StringFieldValue::make("foo")); + expected6.setValue("body", StringFieldValue::make("bar")); + expected6.setValue("aa", IntFieldValue::make(45)); + expected6.setValue("ab", IntFieldValue::make(46)); + expected6.setValue("dd", DoubleFieldValue::make(4.5)); + expected6.setValue("ss", StringFieldValue::make("x45")); TEST_DO(checkEntry(res3, 0, expected5, Timestamp(10))); TEST_DO(checkEntry(res3, 1, expected6, Timestamp(12))); } diff --git a/searchcore/src/tests/proton/documentdb/executor_threading_service/executor_threading_service_test.cpp b/searchcore/src/tests/proton/documentdb/executor_threading_service/executor_threading_service_test.cpp index 934dcdb36e3..46a6419e924 100644 --- a/searchcore/src/tests/proton/documentdb/executor_threading_service/executor_threading_service_test.cpp +++ b/searchcore/src/tests/proton/documentdb/executor_threading_service/executor_threading_service_test.cpp @@ -33,6 +33,7 @@ public: void setup(uint32_t indexing_threads, SharedFieldWriterExecutor shared_field_writer) { service = std::make_unique<ExecutorThreadingService>(_transport.shared(), _transport.transport(), + _transport.clock(), field_writer_executor.get(), nullptr, ThreadingServiceConfig::make(indexing_threads, shared_field_writer)); diff --git a/searchcore/src/tests/proton/index/fusionrunner_test.cpp b/searchcore/src/tests/proton/index/fusionrunner_test.cpp index 38314abd7e5..1b04415b78f 100644 --- a/searchcore/src/tests/proton/index/fusionrunner_test.cpp +++ b/searchcore/src/tests/proton/index/fusionrunner_test.cpp @@ -18,6 +18,7 @@ #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <set> using document::Document; diff --git a/searchcore/src/tests/proton/index/indexcollection_test.cpp b/searchcore/src/tests/proton/index/indexcollection_test.cpp index 70141f057bf..6f6fe3d4e41 100644 --- a/searchcore/src/tests/proton/index/indexcollection_test.cpp +++ b/searchcore/src/tests/proton/index/indexcollection_test.cpp @@ -5,7 +5,7 @@ #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/threadstackexecutor.h> - +#include <vespa/vespalib/util/testclock.h> #include <vespa/log/log.h> LOG_SETUP("indexcollection_test"); @@ -43,6 +43,7 @@ public: std::shared_ptr<IndexSearchable> _source2; std::shared_ptr<IndexSearchable> _fusion_source; vespalib::ThreadStackExecutor _executor; + vespalib::TestClock _clock; std::shared_ptr<IndexSearchable> _warmup; void expect_searchable_can_be_appended(IndexCollection::UP collection) { @@ -76,7 +77,7 @@ public: } IndexCollection::UP create_warmup(const IndexCollection::SP& prev, const IndexCollection::SP& next) { - return std::make_unique<WarmupIndexCollection>(WarmupConfig(1s, false), prev, next, *_warmup, _executor, *this); + return std::make_unique<WarmupIndexCollection>(WarmupConfig(1s, false), prev, next, *_warmup, _executor, _clock.clock(), *this); } void warmupDone(std::shared_ptr<WarmupIndexCollection> current) override { diff --git a/searchcore/src/tests/proton/server/documentretriever_test.cpp b/searchcore/src/tests/proton/server/documentretriever_test.cpp index e631388c9b8..b4236ba4e3e 100644 --- a/searchcore/src/tests/proton/server/documentretriever_test.cpp +++ b/searchcore/src/tests/proton/server/documentretriever_test.cpp @@ -20,6 +20,7 @@ #include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/test/fieldvalue_helpers.h> #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value.h> @@ -66,6 +67,7 @@ using document::StructFieldValue; using document::TensorDataType; using document::TensorFieldValue; using document::WeightedSetFieldValue; +using document::WSetHelper; using search::AttributeFactory; using search::AttributeGuard; using search::AttributeVector; @@ -160,12 +162,12 @@ struct MyDocumentStore : proton::test::DummyDocumentStore { const DocumentType *doc_type = r.getDocumentType(doc_type_name); auto doc = std::make_unique<Document>(*doc_type, doc_id); ASSERT_TRUE(doc); - doc->set(static_field, static_value); - doc->set(dyn_field_i, static_value); - doc->set(dyn_field_s, static_value_s); - doc->set(dyn_field_nai, static_value); - doc->set(dyn_field_nas, static_value_s); - doc->set(zcurve_field, static_zcurve_value); + doc->setValue(static_field, IntFieldValue::make(static_value)); + doc->setValue(dyn_field_i, IntFieldValue::make(static_value)); + doc->setValue(dyn_field_s, StringFieldValue::make(static_value_s)); + doc->setValue(dyn_field_nai, IntFieldValue::make(static_value)); + doc->setValue(dyn_field_nas, StringFieldValue::make(static_value_s)); + doc->setValue(zcurve_field, LongFieldValue::make(static_zcurve_value)); doc->setValue(dyn_field_p, static_value_p); TensorFieldValue tensorFieldValue(tensorDataType); tensorFieldValue = SimpleValue::from_value(*static_tensor); @@ -173,8 +175,8 @@ struct MyDocumentStore : proton::test::DummyDocumentStore { if (_set_position_struct_field) { FieldValue::UP fv = PositionDataType::getInstance().createFieldValue(); auto &pos = dynamic_cast<StructFieldValue &>(*fv); - pos.set(PositionDataType::FIELD_X, 42); - pos.set(PositionDataType::FIELD_Y, 21); + pos.setValue(PositionDataType::FIELD_X, IntFieldValue::make(42)); + pos.setValue(PositionDataType::FIELD_Y, IntFieldValue::make(21)); doc->setValue(doc->getField(position_field), *fv); } @@ -430,10 +432,11 @@ template <typename T> void checkWset(FieldValue::UP wset, T v) { ASSERT_TRUE(wset); auto *wset_val = dynamic_cast<WeightedSetFieldValue *>(wset.get()); + WSetHelper val(*wset_val); ASSERT_TRUE(wset_val); ASSERT_EQUAL(2u, wset_val->size()); - EXPECT_EQUAL(dyn_weight, wset_val->get(v)); - EXPECT_EQUAL(dyn_weight, wset_val->get(v + 1)); + EXPECT_EQUAL(dyn_weight, val.get(v)); + EXPECT_EQUAL(dyn_weight, val.get(v + 1)); } TEST_F("require that attributes are patched into stored document", Fixture) { diff --git a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp index c6c65f45e4b..1feb46cfdb7 100644 --- a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp +++ b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp @@ -1,6 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/child_process.h> +#include <vespa/vespalib/process/process.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/searchcommon/common/schema.h> #include <vespa/searchlib/fef/indexproperties.h> @@ -220,7 +220,7 @@ struct Setup { } bool verify() { generate(); - return vespalib::ChildProcess::run(fmt("%s dir:%s", prog, gen_dir.c_str()).c_str()); + return vespalib::Process::run(fmt("%s dir:%s", prog, gen_dir.c_str())); } void verify_valid(std::initializer_list<std::string> features) { for (const std::string &f: features) { @@ -288,12 +288,12 @@ struct ShadowSetup : Setup { }; TEST_F("print usage", Setup()) { - EXPECT_TRUE(!vespalib::ChildProcess::run(fmt("%s", prog).c_str())); + EXPECT_TRUE(!vespalib::Process::run(fmt("%s", prog))); } TEST_F("setup output directory", Setup()) { - ASSERT_TRUE(vespalib::ChildProcess::run(fmt("rm -rf %s", gen_dir.c_str()).c_str())); - ASSERT_TRUE(vespalib::ChildProcess::run(fmt("mkdir %s", gen_dir.c_str()).c_str())); + ASSERT_TRUE(vespalib::Process::run(fmt("rm -rf %s", gen_dir.c_str()))); + ASSERT_TRUE(vespalib::Process::run(fmt("mkdir %s", gen_dir.c_str()))); } //----------------------------------------------------------------------------- @@ -458,7 +458,7 @@ TEST_F("require that broken fragile model without dry-run passes verification", //----------------------------------------------------------------------------- TEST_F("cleanup files", Setup()) { - ASSERT_TRUE(vespalib::ChildProcess::run(fmt("rm -rf %s", gen_dir.c_str()).c_str())); + ASSERT_TRUE(vespalib::Process::run(fmt("rm -rf %s", gen_dir.c_str()))); } -TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/vespa/searchcore/bmcluster/bm_feed.cpp b/searchcore/src/vespa/searchcore/bmcluster/bm_feed.cpp index 49d1bb1c0a8..56cea8db741 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bm_feed.cpp +++ b/searchcore/src/vespa/searchcore/bmcluster/bm_feed.cpp @@ -65,7 +65,7 @@ BmFeed::make_document(uint32_t n, uint32_t i) const auto id = make_document_id(n, i); auto document = std::make_unique<Document>(*_document_type, id); document->setRepo(*_repo); - document->setFieldValue(_field, std::make_unique<IntFieldValue>(i)); + document->setFieldValue(_field, IntFieldValue::make(i)); return document; } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/document_field_retriever.cpp b/searchcore/src/vespa/searchcore/proton/attribute/document_field_retriever.cpp index 4a220c790c9..00d84e795c7 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/document_field_retriever.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/document_field_retriever.cpp @@ -1,9 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "document_field_retriever.h" -#include <vespa/document/fieldvalue/arrayfieldvalue.h> -#include <vespa/document/fieldvalue/weightedsetfieldvalue.h> -#include <vespa/document/fieldvalue/tensorfieldvalue.h> +#include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/searchcommon/attribute/attributecontent.h> #include <vespa/searchlib/tensor/tensor_attribute.h> #include <vespa/eval/eval/value.h> @@ -31,12 +29,13 @@ namespace proton { namespace { -template <typename T> +template <typename T, typename FT> void -setValue(DocumentIdT lid, - Document &doc, - const document::Field & field, - const IAttributeVector &attr) +setValue(DocumentIdT lid, Document &doc, const document::Field & field, const IAttributeVector &attr); + +template <typename T, typename FT> +void +setValue(DocumentIdT lid, Document &doc, const document::Field & field, const IAttributeVector &attr) { switch (attr.getCollectionType()) { case CollectionType::SINGLE: @@ -44,7 +43,7 @@ setValue(DocumentIdT lid, if ( ! attr.isUndefined(lid) ) { AttributeContent<T> content; content.fill(attr, lid); - doc.set(field, content[0]); + doc.setFieldValue(field, std::make_unique<FT>(content[0])); } else { doc.remove(field); } @@ -65,7 +64,7 @@ setValue(DocumentIdT lid, ArrayFieldValue &array = static_cast<ArrayFieldValue &>(*fv.get()); array.resize(content.size()); for (uint32_t j(0); j < content.size(); ++j) { - array[j] = content[j]; + static_cast<FT &>(array[j]).setValue(content[j]); } doc.setValue(field, *fv); break; @@ -86,8 +85,8 @@ setValue(DocumentIdT lid, wset.resize(content.size()); auto it(wset.begin()); for (uint32_t j(0); j < content.size(); ++j, ++it) { - *it->first = content[j].getValue(); - *it->second = content[j].getWeight(); + static_cast<FT &>(*it->first).setValue(content[j].getValue()); + static_cast<document::IntFieldValue &>(*it->second).setValue(content[j].getWeight()); } doc.setValue(field, *fv); break; @@ -124,27 +123,28 @@ DocumentFieldRetriever::populate(DocumentIdT lid, { switch (attr.getBasicType()) { case BasicType::BOOL: + return setValue<IAttributeVector::largeint_t, document::BoolFieldValue>(lid, doc, field, attr); case BasicType::UINT2: case BasicType::UINT4: case BasicType::INT8: + return setValue<IAttributeVector::largeint_t, document::ByteFieldValue>(lid, doc, field, attr); case BasicType::INT16: + return setValue<IAttributeVector::largeint_t, document::ShortFieldValue>(lid, doc, field, attr); case BasicType::INT32: + return setValue<IAttributeVector::largeint_t, document::IntFieldValue>(lid, doc, field, attr); case BasicType::INT64: - setValue<IAttributeVector::largeint_t>(lid, doc, field, attr); - break; + return setValue<IAttributeVector::largeint_t, document::LongFieldValue>(lid, doc, field, attr); case BasicType::FLOAT: + return setValue<double, document::FloatFieldValue>(lid, doc, field, attr); case BasicType::DOUBLE: - setValue<double>(lid, doc, field, attr); - break; + return setValue<double, document::DoubleFieldValue>(lid, doc, field, attr); case BasicType::STRING: - setValue<const char *>(lid, doc, field, attr); - break; + return setValue<const char *, document::StringFieldValue>(lid, doc, field, attr); case BasicType::PREDICATE: // Predicate attribute doesn't store documents, it only indexes them. break; case BasicType::TENSOR: - setTensorValue(lid, doc, field, attr); - break; + return setTensorValue(lid, doc, field, attr); case BasicType::REFERENCE: // Reference attribute doesn't store full document id. break; diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp index 8489a68af15..0ec27ac419f 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp @@ -177,14 +177,14 @@ MatchEngine::performSearch(search::engine::SearchRequest::Source req) } bool MatchEngine::isOnline() const { - return _nodeUp; + return _nodeUp.load(std::memory_order_relaxed); } void MatchEngine::setNodeUp(bool nodeUp) { - _nodeUp = nodeUp; + _nodeUp.store(nodeUp, std::memory_order_relaxed); } void @@ -192,7 +192,7 @@ MatchEngine::setNodeMaintenance(bool nodeMaintenance) { _nodeMaintenance = nodeMaintenance; if (nodeMaintenance) { - _nodeUp = false; + _nodeUp.store(false, std::memory_order_relaxed); } } diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h index 3d3be775a4a..fcafdc5a5f8 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h @@ -25,7 +25,7 @@ private: HandlerMap<ISearchHandler> _handlers; vespalib::ThreadStackExecutor _executor; vespalib::SimpleThreadBundle::Pool _threadBundlePool; - bool _nodeUp; + std::atomic<bool> _nodeUp; bool _nodeMaintenance; public: diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 37a17fecfc1..e4d75085027 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -186,7 +186,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _baseDir(baseDir + "/" + _docTypeName.toString()), // Only one thread per executor, or performDropFeedView() will fail. _writeServiceConfig(configSnapshot->get_threading_service_config()), - _writeService(shared_service.shared(), shared_service.transport(), shared_service.field_writer(), + _writeService(shared_service.shared(), shared_service.transport(), shared_service.clock(), shared_service.field_writer(), &shared_service.invokeService(), _writeServiceConfig, indexing_thread_stack_size), _initializeThreads(std::move(initializeThreads)), _initConfigSnapshot(), diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp index d3778b4d745..ee2af5b606c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp @@ -5,6 +5,7 @@ #include <vespa/document/datatype/positiondatatype.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> +#include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/fieldset/fieldsets.h> #include <vespa/searchcommon/attribute/attributecontent.h> @@ -157,9 +158,9 @@ positionFromZcurve(int64_t zcurve) { ZCurve::decode(zcurve, &x, &y); FieldValue::UP value = PositionDataType::getInstance().createFieldValue(); - auto *position = static_cast<StructFieldValue *>(value.get()); - position->set(PositionDataType::FIELD_X, x); - position->set(PositionDataType::FIELD_Y, y); + auto *pos = static_cast<StructFieldValue *>(value.get()); + pos->setValue(pos->getField(PositionDataType::FIELD_X), std::make_unique<document::IntFieldValue>(x)); + pos->setValue(pos->getField(PositionDataType::FIELD_Y), std::make_unique<document::IntFieldValue>(y)); return value; } diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp index d6ecc6dd2d3..dd735c75d79 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp @@ -41,12 +41,14 @@ VESPA_THREAD_STACK_TAG(field_writer_executor) } -ExecutorThreadingService::ExecutorThreadingService(vespalib::Executor &sharedExecutor, FNET_Transport & transport, uint32_t num_treads) - : ExecutorThreadingService(sharedExecutor, transport, nullptr, nullptr, ThreadingServiceConfig::make(num_treads)) +ExecutorThreadingService::ExecutorThreadingService(vespalib::Executor &sharedExecutor, FNET_Transport & transport, + const vespalib::Clock & clock, uint32_t num_treads) + : ExecutorThreadingService(sharedExecutor, transport, clock, nullptr, nullptr, ThreadingServiceConfig::make(num_treads)) {} ExecutorThreadingService::ExecutorThreadingService(vespalib::Executor & sharedExecutor, FNET_Transport & transport, + const vespalib::Clock & clock, vespalib::ISequencedTaskExecutor * field_writer, vespalib::InvokeService * invokerService, const ThreadingServiceConfig & cfg, @@ -54,6 +56,7 @@ ExecutorThreadingService::ExecutorThreadingService(vespalib::Executor & sharedEx : _sharedExecutor(sharedExecutor), _transport(transport), + _clock(clock), _masterExecutor(1, stackSize, CpuUsage::wrap(master_executor, CpuUsage::Category::WRITE)), _shared_field_writer(cfg.shared_field_writer()), _master_task_limit(cfg.master_task_limit()), diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h index 77bb9042198..1179c88ef76 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h +++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h @@ -22,6 +22,7 @@ private: using Registration = std::unique_ptr<vespalib::IDestructorCallback>; vespalib::Executor & _sharedExecutor; FNET_Transport & _transport; + const vespalib::Clock & _clock; vespalib::ThreadStackExecutor _masterExecutor; ThreadingServiceConfig::SharedFieldWriterExecutor _shared_field_writer; std::atomic<uint32_t> _master_task_limit; @@ -43,10 +44,12 @@ public: /** * Convenience constructor used in unit tests. */ - ExecutorThreadingService(vespalib::Executor& sharedExecutor, FNET_Transport & transport, uint32_t num_treads = 1); + ExecutorThreadingService(vespalib::Executor& sharedExecutor, FNET_Transport & transport, + const vespalib::Clock & clock, uint32_t num_treads = 1); ExecutorThreadingService(vespalib::Executor& sharedExecutor, FNET_Transport & transport, + const vespalib::Clock & clock, vespalib::ISequencedTaskExecutor* field_writer, vespalib::InvokeService * invokeService, const ThreadingServiceConfig& cfg, @@ -82,6 +85,7 @@ public: vespalib::ISequencedTaskExecutor &indexFieldWriter() override; vespalib::ISequencedTaskExecutor &attributeFieldWriter() override; FNET_Transport &transport() override { return _transport; } + const vespalib::Clock &clock() const override { return _clock; } ExecutorThreadingServiceStats getStats(); }; diff --git a/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h b/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h index f728ab5f025..16b466a2275 100644 --- a/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h +++ b/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h @@ -50,6 +50,7 @@ public: return _shared; } FNET_Transport & transport() override { return _service.transport(); } + const vespalib::Clock & clock() const override { return _service.clock(); } vespalib::ISequencedTaskExecutor &indexFieldInverter() override { return _indexFieldInverter; } diff --git a/searchcore/src/vespa/searchcore/proton/test/transport_helper.cpp b/searchcore/src/vespa/searchcore/proton/test/transport_helper.cpp index 43e267805da..0731a3429b1 100644 --- a/searchcore/src/vespa/searchcore/proton/test/transport_helper.cpp +++ b/searchcore/src/vespa/searchcore/proton/test/transport_helper.cpp @@ -5,13 +5,15 @@ #include <vespa/fastos/thread.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/vespalib/util/testclock.h> #include <vespa/searchcore/proton/server/executorthreadingservice.h> namespace proton { Transport::Transport() : _threadPool(std::make_unique<FastOS_ThreadPool>(64_Ki)), - _transport(std::make_unique<FNET_Transport>()) + _transport(std::make_unique<FNET_Transport>()), + _clock(std::make_unique<vespalib::TestClock>()) { _transport->Start(_threadPool.get()); } @@ -20,6 +22,10 @@ Transport::~Transport() { shutdown(); } +const vespalib::Clock & +Transport::clock() const { + return _clock->clock(); +} void Transport::shutdown() { _transport->ShutDown(true); @@ -39,7 +45,7 @@ TransportAndExecutor::shutdown() { TransportAndExecutorService::TransportAndExecutorService(size_t num_threads) : TransportAndExecutor(num_threads), - _writeService(std::make_unique<ExecutorThreadingService>(shared(), transport())) + _writeService(std::make_unique<ExecutorThreadingService>(shared(), transport(), clock())) {} TransportAndExecutorService::~TransportAndExecutorService() = default; diff --git a/searchcore/src/vespa/searchcore/proton/test/transport_helper.h b/searchcore/src/vespa/searchcore/proton/test/transport_helper.h index 81610193743..8ec4f50e3f0 100644 --- a/searchcore/src/vespa/searchcore/proton/test/transport_helper.h +++ b/searchcore/src/vespa/searchcore/proton/test/transport_helper.h @@ -5,6 +5,8 @@ class FastOS_ThreadPool; +namespace vespalib { class TestClock; } + namespace proton { class ExecutorThreadingService; @@ -18,10 +20,12 @@ public: virtual ~Transport(); FNET_Transport & transport() { return *_transport; } FastOS_ThreadPool & threadPool() { return *_threadPool; } + const vespalib::Clock & clock() const; virtual void shutdown(); private: std::unique_ptr<FastOS_ThreadPool> _threadPool; std::unique_ptr<FNET_Transport> _transport; + std::unique_ptr<vespalib::TestClock> _clock; }; class TransportAndExecutor : public Transport { diff --git a/searchcorespi/src/vespa/searchcorespi/index/imemoryindex.h b/searchcorespi/src/vespa/searchcorespi/index/imemoryindex.h index 16feeb2ce24..67d6e034080 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/imemoryindex.h +++ b/searchcorespi/src/vespa/searchcorespi/index/imemoryindex.h @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/document/fieldvalue/document.h> #include <vespa/searchcommon/common/schema.h> #include <vespa/searchcorespi/index/indexsearchable.h> #include <vespa/searchlib/common/serialnum.h> @@ -9,7 +8,7 @@ #include <vespa/vespalib/util/memoryusage.h> namespace vespalib { class IDestructorCallback; } - +namespace document { class Document; } namespace searchcorespi::index { /** diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index 1153a09d09f..20aba93d4d7 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -9,6 +9,7 @@ #include "indexreadutilities.h" #include "indexwriteutilities.h" #include "index_disk_dir.h" +#include <vespa/document/fieldvalue/document.h> #include <vespa/searchcorespi/flush/lambdaflushtask.h> #include <vespa/searchlib/common/i_flush_token.h> #include <vespa/searchlib/index/schemautil.h> @@ -431,7 +432,8 @@ IndexMaintainer::swapInNewIndex(LockGuard & guard, LOG(debug, "Warming up a disk index."); indexes = std::make_shared<WarmupIndexCollection> (_warmupConfig, getLeaf(guard, _source_list, true), indexes, - static_cast<IDiskIndex &>(source), _ctx.getWarmupExecutor(), *this); + static_cast<IDiskIndex &>(source), _ctx.getWarmupExecutor(), + _ctx.getThreadingService().clock(), *this); } else { LOG(debug, "No warmup needed as it is a memory index that is mapped in."); } diff --git a/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h b/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h index 60997d6666b..c325d5ded11 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h +++ b/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h @@ -5,7 +5,10 @@ class FNET_Transport; -namespace vespalib { class ISequencedTaskExecutor; } +namespace vespalib { + class ISequencedTaskExecutor; + class Clock; +} namespace searchcorespi::index { /** @@ -76,6 +79,7 @@ struct IThreadingService virtual vespalib::ThreadExecutor &summary() = 0; virtual vespalib::Executor &shared() = 0; virtual FNET_Transport &transport() = 0; + virtual const vespalib::Clock &clock() const = 0; virtual vespalib::ISequencedTaskExecutor &indexFieldInverter() = 0; virtual vespalib::ISequencedTaskExecutor &indexFieldWriter() = 0; virtual vespalib::ISequencedTaskExecutor &attributeFieldWriter() = 0; diff --git a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp index b9cbdab1c0a..bf437dd7ee3 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp @@ -1,10 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #include "warmupindexcollection.h" #include "idiskindex.h" #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/query/tree/termnodes.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/stllike/hash_set.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/eval/eval/value.h> #include <thread> #include <vespa/log/log.h> @@ -31,12 +34,14 @@ WarmupIndexCollection::WarmupIndexCollection(const WarmupConfig & warmupConfig, ISearchableIndexCollection::SP next, IndexSearchable & warmup, vespalib::Executor & executor, + const vespalib::Clock & clock, IWarmupDone & warmupDone) : _warmupConfig(warmupConfig), _prev(std::move(prev)), _next(std::move(next)), _warmup(warmup), _executor(executor), + _clock(clock), _warmupDone(warmupDone), _warmupEndTime(vespalib::steady_clock::now() + warmupConfig.getDuration()), _handledTerms(std::make_unique<FieldTermMap>()), @@ -223,12 +228,21 @@ WarmupIndexCollection::drainPending() { _pendingTasks.waitForZeroRefCount(); } +WarmupIndexCollection::WarmupRequestContext::WarmupRequestContext(const vespalib::Clock & clock) + : _doom(clock, vespalib::steady_time::max(), vespalib::steady_time::max(), false) +{} +WarmupIndexCollection::WarmupRequestContext::~WarmupRequestContext() = default; + +std::unique_ptr<vespalib::eval::Value> +WarmupIndexCollection::WarmupRequestContext::get_query_tensor(const vespalib::string&) const { + return {}; +} WarmupIndexCollection::WarmupTask::WarmupTask(std::unique_ptr<MatchData> md, std::shared_ptr<WarmupIndexCollection> warmup) : _warmup(std::move(warmup)), _retainGuard(_warmup->_pendingTasks), _matchData(std::move(md)), _bluePrint(), - _requestContext() + _requestContext(_warmup->_clock) { } diff --git a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h index b0b2952bee8..c2d70b4fd5c 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h +++ b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h @@ -4,10 +4,11 @@ #include "isearchableindexcollection.h" #include "warmupconfig.h" -#include <vespa/vespalib/util/threadexecutor.h> +#include <vespa/searchlib/attribute/attribute_blueprint_params.h> +#include <vespa/vespalib/util/doom.h> +#include <vespa/vespalib/util/executor.h> #include <vespa/vespalib/util/monitored_refcount.h> #include <vespa/vespalib/util/retain_guard.h> -#include <vespa/searchlib/queryeval/fake_requestcontext.h> namespace searchcorespi { @@ -34,6 +35,7 @@ public: ISearchableIndexCollection::SP next, IndexSearchable & warmup, vespalib::Executor & executor, + const vespalib::Clock & clock, IWarmupDone & warmupDone); ~WarmupIndexCollection() override; // Implements IIndexCollection @@ -70,8 +72,22 @@ public: void drainPending(); private: typedef search::fef::MatchData MatchData; - typedef search::queryeval::FakeRequestContext FakeRequestContext; typedef vespalib::Executor::Task Task; + class WarmupRequestContext : public IRequestContext { + using IAttributeVector = search::attribute::IAttributeVector; + using AttributeBlueprintParams = search::attribute::AttributeBlueprintParams; + public: + WarmupRequestContext(const vespalib::Clock & clock); + ~WarmupRequestContext() override; + const vespalib::Doom & getDoom() const override { return _doom; } + const IAttributeVector *getAttribute(const vespalib::string &) const override { return nullptr; } + const IAttributeVector *getAttributeStableEnum(const vespalib::string &) const override { return nullptr; } + std::unique_ptr<vespalib::eval::Value> get_query_tensor(const vespalib::string&) const override; + const AttributeBlueprintParams& get_attribute_blueprint_params() const override { return _params; } + private: + const vespalib::Doom _doom; + const AttributeBlueprintParams _params; + }; class WarmupTask : public Task { public: WarmupTask(std::unique_ptr<MatchData> md, std::shared_ptr<WarmupIndexCollection> warmup); @@ -90,7 +106,7 @@ private: vespalib::RetainGuard _retainGuard; std::unique_ptr<MatchData> _matchData; Blueprint::UP _bluePrint; - FakeRequestContext _requestContext; + WarmupRequestContext _requestContext; }; void fireWarmup(Task::UP task); @@ -101,6 +117,7 @@ private: ISearchableIndexCollection::SP _next; IndexSearchable & _warmup; vespalib::Executor & _executor; + const vespalib::Clock & _clock; IWarmupDone & _warmupDone; vespalib::steady_time _warmupEndTime; std::mutex _lock; diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index ced2517ff9f..5a534562d32 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -943,7 +943,6 @@ "public final java.lang.String tag()", "public final java.util.List tagCommaLeadingList()", "public final com.yahoo.searchlib.rankingexpression.rule.ExpressionNode constantPrimitive(boolean)", - "public final com.yahoo.searchlib.rankingexpression.evaluation.Value primitiveValue()", "public final com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode tensorValueBody(com.yahoo.tensor.TensorType, java.util.List)", "public final com.yahoo.tensor.functions.DynamicTensor mappedTensorValueBody(com.yahoo.tensor.TensorType)", "public final com.yahoo.tensor.functions.DynamicTensor mixedTensorValueBody(com.yahoo.tensor.TensorType, java.util.List)", @@ -1086,6 +1085,8 @@ "public static final int MIN", "public static final int PROD", "public static final int SUM", + "public static final int TRUE", + "public static final int FALSE", "public static final int IDENTIFIER", "public static final int SINGLE_LINE_COMMENT", "public static final int DEFAULT", diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index fcac0be8f85..07747a66892 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -137,7 +137,7 @@ setFieldValue(Document &doc, const vespalib::string &fieldName, const vespalib::string &fieldString) { std::unique_ptr<StringFieldValue> fieldValue = - std::make_unique<StringFieldValue>(fieldString); + StringFieldValue::make(fieldString); document::FixedTypeRepo repo(*doc.getRepo(), doc.getType()); tokenizeStringFieldValue(repo, *fieldValue); doc.setFieldValue(doc.getField(fieldName), std::move(fieldValue)); diff --git a/searchlib/src/apps/vespa-ranking-expression-analyzer/vespa-ranking-expression-analyzer.cpp b/searchlib/src/apps/vespa-ranking-expression-analyzer/vespa-ranking-expression-analyzer.cpp index 8db85559f49..d4c11d5ac2f 100644 --- a/searchlib/src/apps/vespa-ranking-expression-analyzer/vespa-ranking-expression-analyzer.cpp +++ b/searchlib/src/apps/vespa-ranking-expression-analyzer/vespa-ranking-expression-analyzer.cpp @@ -331,7 +331,6 @@ State::~State() {} struct MyApp : public FastOS_Application { int Main() override; int usage(); - virtual bool useProcessStarter() const override { return false; } }; int diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/BooleanValue.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/BooleanValue.java index 770be98c739..49f267ca522 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/BooleanValue.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/BooleanValue.java @@ -15,7 +15,7 @@ public class BooleanValue extends DoubleCompatibleValue { * Create a boolean value which is frozen at the outset. */ public static BooleanValue frozen(boolean value) { - BooleanValue booleanValue=new BooleanValue(value); + BooleanValue booleanValue = new BooleanValue(value); booleanValue.freeze(); return booleanValue; } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ConstantNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ConstantNode.java index 46e833197f9..a305c9c76af 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ConstantNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ConstantNode.java @@ -2,6 +2,7 @@ package com.yahoo.searchlib.rankingexpression.rule; import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.searchlib.rankingexpression.evaluation.BooleanValue; import com.yahoo.searchlib.rankingexpression.evaluation.Context; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.tensor.TensorType; diff --git a/searchlib/src/main/javacc/RankingExpressionParser.jj b/searchlib/src/main/javacc/RankingExpressionParser.jj index 865820320d8..ebe1e048247 100755 --- a/searchlib/src/main/javacc/RankingExpressionParser.jj +++ b/searchlib/src/main/javacc/RankingExpressionParser.jj @@ -153,6 +153,8 @@ TOKEN : <MIN: "min"> | <PROD: "prod"> | <SUM: "sum"> | + <TRUE: "true"> | + <FALSE: "false"> | <IDENTIFIER: (["A"-"Z","a"-"z","0"-"9","_","@"](["A"-"Z","a"-"z","0"-"9","_","@","$"])*)> } @@ -797,7 +799,9 @@ String identifier() : func = binaryFunctionName() { return func.toString(); } | <IF> { return token.image; } | <IN> { return token.image; } | - <IDENTIFIER> { return token.image; } + <IDENTIFIER> { return token.image; } | + <TRUE> { return token.image; } | + <FALSE> { return token.image; } } List<String> identifierList() : @@ -854,25 +858,14 @@ ExpressionNode constantPrimitive(boolean negate) : ( <INTEGER> { value = token.image; } | <FLOAT> { value = token.image; } ) { node = new ConstantNode(Value.parse(negate ? ("-" + value) : value)); } | - <STRING> { - value = token.image; - node = new ConstantNode(Value.parse(value)); + ( <STRING> | <TRUE> | <FALSE> ) { + node = new ConstantNode(Value.parse(token.image)); if (negate) node = new NegativeNode(node); } ) { return node; } } -Value primitiveValue() : -{ - String sign = ""; -} -{ - ( <SUB> { sign = "-";} ) ? - ( <INTEGER> | <FLOAT> | <STRING> ) - { return Value.parse(sign + token.image); } -} - TensorFunctionNode tensorValueBody(TensorType type, List dimensionOrder) : { DynamicTensor dynamicTensor; diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java index efa98fba2eb..19e32c23234 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java @@ -133,6 +133,10 @@ public class EvaluationTestCase { public void testBooleanEvaluation() { EvaluationTester tester = new EvaluationTester(); + // literals + tester.assertEvaluates(false, "false"); + tester.assertEvaluates(true, "true"); + // and tester.assertEvaluates(false, "0 && 0"); tester.assertEvaluates(false, "0 && 1"); diff --git a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp index 072efb06a07..c08aa7ce9fc 100644 --- a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp +++ b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp @@ -8,6 +8,7 @@ #include <vespa/searchlib/common/bitvector.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/datatype/documenttype.h> +#include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/io/fileutil.h> @@ -26,6 +27,7 @@ using document::Document; using document::DocumentId; using document::DocumentType; using document::DocumentTypeRepo; +using document::StringFieldValue; using vespalib::compression::CompressionConfig; using vespalib::asciistream; using index::DummyFileHeaderContext; @@ -66,9 +68,9 @@ makeDoc(const DocumentTypeRepo &repo, uint32_t i, bool before) mainstr << (j + i * 1000) << " "; } mainstr << " and end field"; - doc->set("main", mainstr.c_str()); + doc->setValue("main", StringFieldValue::make(mainstr.str())); if (!before) { - doc->set("extra", "foo"); + doc->setValue("extra", StringFieldValue::make("foo")); } return doc; @@ -160,7 +162,7 @@ MyRewriteVisitor::visit(uint32_t lid, const std::shared_ptr<Document> &doc) Document::UP expDoc(makeDoc(_repo, lid, _before)); EXPECT_TRUE(*expDoc == *doc); _valid->setBitAndMaintainCount(lid); - doc->set("extra", "foo"); + doc->setValue("extra", StringFieldValue::make("foo")); } diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp index 378babb6ee1..bb2d0177dd9 100644 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp @@ -4,6 +4,7 @@ #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/datatype/documenttype.h> +#include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/searchlib/docstore/chunkformats.h> #include <vespa/searchlib/docstore/logdocumentstore.h> @@ -19,6 +20,7 @@ #include <iomanip> using document::BucketId; +using document::StringFieldValue; using namespace search::docstore; using namespace search; using namespace vespalib::alloc; @@ -429,9 +431,9 @@ makeDoc(const DocumentTypeRepo &repo, uint32_t i, bool extra_field, size_t numRe mainstr << (j + i * 1000) << " "; } mainstr << " and end field"; - doc->set("main", mainstr.c_str()); + doc->setValue("main", StringFieldValue::make(mainstr.str())); if (extra_field) { - doc->set("extra", "foo"); + doc->setValue("extra", StringFieldValue::make("foo")); } return doc; } diff --git a/searchlib/src/tests/grouping/grouping_serialization_test.cpp b/searchlib/src/tests/grouping/grouping_serialization_test.cpp index ea96c698a02..39a5feab111 100644 --- a/searchlib/src/tests/grouping/grouping_serialization_test.cpp +++ b/searchlib/src/tests/grouping/grouping_serialization_test.cpp @@ -7,6 +7,7 @@ #include <vespa/searchlib/expression/getdocidnamespacespecificfunctionnode.h> #include <vespa/searchlib/expression/getymumchecksumfunctionnode.h> #include <vespa/searchlib/expression/documentfieldnode.h> +#include <vespa/document/base/documentid.h> #include <vespa/vespalib/testkit/test_kit.h> #include <fstream> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/attribute/atomic_utils.h b/searchlib/src/vespa/searchlib/attribute/atomic_utils.h new file mode 100644 index 00000000000..48914de8942 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/atomic_utils.h @@ -0,0 +1,34 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace vespalib::datastore { + +class AtomicEntryRef; +class EntryRef; + +} + +namespace search::attribute::atomic_utils { + +/* + * Helper class to map from atomic value to non-atomic value, e.g. + * from AtomicEntryRef to EntryRef. + */ +template <typename MaybeAtomicValue> +class NonAtomicValue { +public: + using type = MaybeAtomicValue; +}; + +template <> +class NonAtomicValue<vespalib::datastore::AtomicEntryRef> +{ +public: + using type = vespalib::datastore::EntryRef; +}; + +template <class MaybeAtomicValue> +using NonAtomicValue_t = typename NonAtomicValue<MaybeAtomicValue>::type; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.cpp b/searchlib/src/vespa/searchlib/attribute/load_utils.cpp index f71222e6faa..707cdc6af76 100644 --- a/searchlib/src/vespa/searchlib/attribute/load_utils.cpp +++ b/searchlib/src/vespa/searchlib/attribute/load_utils.cpp @@ -12,6 +12,7 @@ using search::multivalue::Value; using search::multivalue::WeightedValue; +using vespalib::datastore::AtomicEntryRef; namespace search::attribute { @@ -84,15 +85,15 @@ template uint32_t loadFromEnumeratedMultiValue(MultiValueMapping<Value<ValueType #define INSTANTIATE_WSET(ValueType, Saver) \ template uint32_t loadFromEnumeratedMultiValue(MultiValueMapping<WeightedValue<ValueType>> &, ReaderBase &, vespalib::ConstArrayRef<ValueType>, vespalib::ConstArrayRef<uint32_t>, Saver) #define INSTANTIATE_SINGLE(ValueType, Saver) \ -template void loadFromEnumeratedSingleValue(vespalib::RcuVectorBase<ValueType> &, vespalib::GenerationHolder &, ReaderBase &, vespalib::ConstArrayRef<ValueType>, vespalib::ConstArrayRef<uint32_t>, Saver) +template void loadFromEnumeratedSingleValue(vespalib::RcuVectorBase<ValueType> &, vespalib::GenerationHolder &, ReaderBase &, vespalib::ConstArrayRef<atomic_utils::NonAtomicValue_t<ValueType>>, vespalib::ConstArrayRef<uint32_t>, Saver) #define INSTANTIATE_SINGLE_ARRAY_WSET(ValueType, Saver) \ INSTANTIATE_SINGLE(ValueType, Saver); \ -INSTANTIATE_ARRAY(ValueType, Saver); \ -INSTANTIATE_WSET(ValueType, Saver) +INSTANTIATE_ARRAY(atomic_utils::NonAtomicValue_t<ValueType>, Saver); \ +INSTANTIATE_WSET(atomic_utils::NonAtomicValue_t<ValueType>, Saver) #define INSTANTIATE_ENUM(Saver) \ -INSTANTIATE_SINGLE_ARRAY_WSET(IEnumStore::Index, Saver) +INSTANTIATE_SINGLE_ARRAY_WSET(AtomicEntryRef, Saver) #define INSTANTIATE_VALUE(ValueType) \ INSTANTIATE_SINGLE_ARRAY_WSET(ValueType, NoSaveLoadedEnum) diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.h b/searchlib/src/vespa/searchlib/attribute/load_utils.h index a9a41b8eaeb..f9f933f3726 100644 --- a/searchlib/src/vespa/searchlib/attribute/load_utils.h +++ b/searchlib/src/vespa/searchlib/attribute/load_utils.h @@ -2,10 +2,18 @@ #pragma once +#include "atomic_utils.h" #include "attributevector.h" #include "readerbase.h" #include <vespa/vespalib/util/arrayref.h> +namespace vespalib::datastore { + +class AtomicEntryRef; +class EntryRef; + +} + namespace search::attribute { /** @@ -51,7 +59,7 @@ void loadFromEnumeratedSingleValue(Vector &vector, vespalib::GenerationHolder &genHolder, ReaderBase &attrReader, - vespalib::ConstArrayRef<typename Vector::ValueType> enumValueToValueMap, + vespalib::ConstArrayRef<atomic_utils::NonAtomicValue_t<typename Vector::ValueType>> enumValueToValueMap, vespalib::ConstArrayRef<uint32_t> enum_value_remapping, Saver saver) __attribute((noinline)); diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.hpp b/searchlib/src/vespa/searchlib/attribute/load_utils.hpp index 50566b36e75..74126299919 100644 --- a/searchlib/src/vespa/searchlib/attribute/load_utils.hpp +++ b/searchlib/src/vespa/searchlib/attribute/load_utils.hpp @@ -54,10 +54,12 @@ void loadFromEnumeratedSingleValue(Vector &vector, vespalib::GenerationHolder &genHolder, ReaderBase &attrReader, - vespalib::ConstArrayRef<typename Vector::ValueType> enumValueToValueMap, + vespalib::ConstArrayRef<atomic_utils::NonAtomicValue_t<typename Vector::ValueType>> enumValueToValueMap, vespalib::ConstArrayRef<uint32_t> enum_value_remapping, Saver saver) { + using ValueType = typename Vector::ValueType; + using NonAtomicValueType = atomic_utils::NonAtomicValue_t<ValueType>; uint32_t numDocs = attrReader.getEnumCount(); genHolder.clearHoldLists(); vector.reset(); @@ -68,7 +70,11 @@ loadFromEnumeratedSingleValue(Vector &vector, if (!enum_value_remapping.empty()) { enumValue = enum_value_remapping[enumValue]; } - vector.push_back(enumValueToValueMap[enumValue]); + if constexpr (std::is_same_v<ValueType, NonAtomicValueType>) { + vector.push_back(enumValueToValueMap[enumValue]); + } else { + vector.push_back(ValueType(enumValueToValueMap[enumValue])); + } saver.save(enumValue, doc, 1); } } diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp index cffd52f3dc4..b5de652fbe2 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp @@ -32,7 +32,7 @@ AttributeVector::DocId SingleValueEnumAttributeBase::addDoc(bool &incGeneration) { incGeneration = _enumIndices.isFull(); - _enumIndices.push_back(IEnumStore::Index()); + _enumIndices.push_back(AtomicEntryRef()); return _enumIndices.size() - 1; } @@ -41,7 +41,12 @@ SingleValueEnumAttributeBase::EnumIndexCopyVector SingleValueEnumAttributeBase::getIndicesCopy(uint32_t size) const { assert(size <= _enumIndices.size()); - return EnumIndexCopyVector(&_enumIndices[0], &_enumIndices[0] + size); + EnumIndexCopyVector result; + result.reserve(size); + for (uint32_t lid = 0; lid < size; ++lid) { + result.push_back(_enumIndices[lid].load_relaxed()); + } + return result; } void @@ -54,11 +59,11 @@ SingleValueEnumAttributeBase::remap_enum_store_refs(const EnumIndexRemapper& rem v.logEnumStoreEvent("reenumerate", "start"); auto& filter = remapper.get_entry_ref_filter(); for (uint32_t i = 0; i < _enumIndices.size(); ++i) { - EnumIndex ref = _enumIndices[i]; + EnumIndex ref = _enumIndices[i].load_relaxed(); if (ref.valid() && filter.has(ref)) { ref = remapper.remap(ref); } - new_indexes.push_back_fast(ref); + new_indexes.push_back_fast(AtomicEntryRef(ref)); } v.logEnumStoreEvent("compactfixup", "drain"); { diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.h b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.h index 8fcef670fb0..4ce55902d7f 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.h @@ -17,24 +17,25 @@ class ReaderBase; */ class SingleValueEnumAttributeBase { protected: + using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; + using AtomicEntryRefVector = vespalib::RcuVectorBase<AtomicEntryRef>; using DocId = AttributeVector::DocId; using EnumHandle = AttributeVector::EnumHandle; using EnumIndex = IEnumStore::Index; - using EnumIndexVector = vespalib::RcuVectorBase<EnumIndex>; using EnumIndexRemapper = IEnumStore::EnumIndexRemapper; using GenerationHolder = vespalib::GenerationHolder; public: using EnumIndexCopyVector = vespalib::Array<EnumIndex>; - IEnumStore::Index getEnumIndex(DocId docId) const { return _enumIndices[docId]; } - EnumHandle getE(DocId doc) const { return _enumIndices[doc].ref(); } + IEnumStore::Index getEnumIndex(DocId docId) const { return _enumIndices[docId].load_acquire(); } + EnumHandle getE(DocId doc) const { return _enumIndices[doc].load_acquire().ref(); } protected: SingleValueEnumAttributeBase(const attribute::Config & c, GenerationHolder &genHolder, const vespalib::alloc::Alloc& initial_alloc); ~SingleValueEnumAttributeBase(); AttributeVector::DocId addDoc(bool & incGeneration); - EnumIndexVector _enumIndices; + AtomicEntryRefVector _enumIndices; EnumIndexCopyVector getIndicesCopy(uint32_t size) const; void remap_enum_store_refs(const EnumIndexRemapper& remapper, AttributeVector& v); diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp index 56aa6672696..bb454a8b0d4 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp @@ -53,9 +53,9 @@ SingleValueEnumAttribute<B>::addDoc(DocId & doc) if (doc > 0u) { // Make sure that a valid value(magic default) is referenced, // even between addDoc and commit(). - if (_enumIndices[0].valid()) { + if (_enumIndices[0].load_relaxed().valid()) { _enumIndices[doc] = _enumIndices[0]; - this->_enumStore.inc_ref_count(_enumIndices[0]); + this->_enumStore.inc_ref_count(_enumIndices[0].load_relaxed()); } } this->incNumDocs(); @@ -166,7 +166,7 @@ template <typename B> void SingleValueEnumAttribute<B>::applyUpdateValueChange(const Change& c, EnumStoreBatchUpdater& updater) { - EnumIndex oldIdx = _enumIndices[c._doc]; + EnumIndex oldIdx = _enumIndices[c._doc].load_relaxed(); EnumIndex newIdx; if (c.has_entry_ref()) { newIdx = EnumIndex(vespalib::datastore::EntryRef(c.get_entry_ref())); @@ -204,7 +204,7 @@ SingleValueEnumAttribute<B>::updateEnumRefCounts(const Change& c, EnumIndex newI EnumStoreBatchUpdater& updater) { updater.inc_ref_count(newIdx); - _enumIndices[c._doc] = newIdx; + _enumIndices[c._doc].store_release(newIdx); if (oldIdx.valid()) { updater.dec_ref_count(oldIdx); } @@ -220,7 +220,7 @@ SingleValueEnumAttribute<B>::fillValues(LoadedVector & loaded) _enumIndices.reset(); _enumIndices.unsafe_reserve(numDocs); for (DocId doc = 0; doc < numDocs; ++doc, loaded.next()) { - _enumIndices.push_back(loaded.read().getEidx()); + _enumIndices.push_back(AtomicEntryRef(loaded.read().getEidx())); } } } @@ -296,7 +296,7 @@ SingleValueEnumAttribute<B>::clearDocs(DocId lidLow, DocId lidLimit) assert(lidLow <= lidLimit); assert(lidLimit <= this->getNumDocs()); for (DocId lid = lidLow; lid < lidLimit; ++lid) { - if (_enumIndices[lid] != vespalib::datastore::EntryRef(e)) { + if (_enumIndices[lid].load_relaxed() != vespalib::datastore::EntryRef(e)) { this->clearDoc(lid); } } diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h index 4383527a4a4..9ef2d680f05 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h @@ -103,7 +103,7 @@ public: // Attribute read API //------------------------------------------------------------------------- T get(DocId doc) const override { - return this->_enumStore.get_value(this->_enumIndices[doc]); + return this->_enumStore.get_value(this->_enumIndices[doc].load_acquire()); } largeint_t getInt(DocId doc) const override { return static_cast<largeint_t>(get(doc)); diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp index c6b720ba14e..215123e9b5b 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp @@ -50,7 +50,7 @@ template <typename B> void SingleValueNumericEnumAttribute<B>::applyArithmeticValueChange(const Change& c, EnumStoreBatchUpdater& updater) { - EnumIndex oldIdx = this->_enumIndices[c._doc]; + EnumIndex oldIdx = this->_enumIndices[c._doc].load_relaxed(); EnumIndex newIdx; T newValue = this->template applyArithmetic<T, typename Change::DataType>(get(c._doc), c._data.getArithOperand(), c._type); this->_enumStore.find_index(newValue, newIdx); diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp index 4d37171e151..b87dadb5429 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp @@ -64,7 +64,7 @@ makePostingChange(const vespalib::datastore::EntryComparator &cmpa, { for (const auto& elem : currEnumIndices) { uint32_t docId = elem.first; - EnumIndex oldIdx = this->_enumIndices[docId]; + EnumIndex oldIdx = this->_enumIndices[docId].load_relaxed(); EnumIndex newIdx = elem.second; // add new posting @@ -97,7 +97,7 @@ SingleValueNumericPostingAttribute<B>::applyValueChanges(EnumStoreBatchUpdater& if (enumIter != currEnumIndices.end()) { oldIdx = enumIter->second; } else { - oldIdx = this->_enumIndices[change._doc]; + oldIdx = this->_enumIndices[change._doc].load_relaxed(); } if (change._type == ChangeBase::UPDATE) { diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h index cd0fe5cfb9a..45bb007e737 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h @@ -45,7 +45,7 @@ public: //------------------------------------------------------------------------- bool isUndefined(DocId doc) const override { return get(doc)[0] == '\0'; } const char * get(DocId doc) const override { - return this->_enumStore.get_value(this->_enumIndices[doc]); + return this->_enumStore.get_value(this->_enumIndices[doc].load_acquire()); } std::vector<EnumHandle> findFoldedEnums(const char *value) const override { return this->_enumStore.find_folded_enums(value); @@ -95,7 +95,7 @@ public: int32_t onFind(DocId doc, int32_t elemId) const override { if ( elemId != 0) return -1; const SingleValueStringAttributeT<B> & attr(static_cast<const SingleValueStringAttributeT<B> &>(attribute())); - return isMatch(attr._enumStore.get_value(attr._enumIndices[doc])) ? 0 : -1; + return isMatch(attr._enumStore.get_value(attr._enumIndices[doc].load_acquire())) ? 0 : -1; } }; diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp index 4670ee075fe..58c62e60efe 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp @@ -64,7 +64,7 @@ makePostingChange(const vespalib::datastore::EntryComparator &cmpa, { for (const auto& elem : currEnumIndices) { uint32_t docId = elem.first; - EnumIndex oldIdx = this->_enumIndices[docId]; + EnumIndex oldIdx = this->_enumIndices[docId].load_relaxed(); EnumIndex newIdx = elem.second; // add new posting @@ -98,7 +98,7 @@ SingleValueStringPostingAttributeT<B>::applyValueChanges(EnumStoreBatchUpdater& if (enumIter != currEnumIndices.end()) { oldIdx = enumIter->second; } else { - oldIdx = this->_enumIndices[change._doc]; + oldIdx = this->_enumIndices[change._doc].load_relaxed(); } if (change._type == ChangeBase::UPDATE) { applyUpdateValueChange(change, enumStore, currEnumIndices); diff --git a/searchlib/src/vespa/searchlib/expression/documentaccessornode.h b/searchlib/src/vespa/searchlib/expression/documentaccessornode.h index 0efe7d023f6..453b8f82bf0 100644 --- a/searchlib/src/vespa/searchlib/expression/documentaccessornode.h +++ b/searchlib/src/vespa/searchlib/expression/documentaccessornode.h @@ -2,12 +2,14 @@ #pragma once #include "expressionnode.h" -#include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/objects/objectoperation.h> #include <vespa/vespalib/objects/objectpredicate.h> -namespace search { -namespace expression { +namespace document { + class DocumentType; + class Document; +} +namespace search::expression { class DocumentAccessorNode : public ExpressionNode { @@ -33,5 +35,3 @@ private: }; } -} - diff --git a/searchlib/src/vespa/searchlib/expression/documentfieldnode.h b/searchlib/src/vespa/searchlib/expression/documentfieldnode.h index b85ba0f4e9b..fd3923bd4a0 100644 --- a/searchlib/src/vespa/searchlib/expression/documentfieldnode.h +++ b/searchlib/src/vespa/searchlib/expression/documentfieldnode.h @@ -5,6 +5,7 @@ #include "resultnode.h" #include "resultvector.h" #include <vespa/document/fieldvalue/iteratorhandler.h> +#include <vespa/document/base/fieldpath.h> namespace search::expression { diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h index 1c1998fb423..5a7c8d0f7bc 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h @@ -4,7 +4,6 @@ #include "i_field_index_remove_listener.h" #include <vespa/document/annotation/span.h> -#include <vespa/document/fieldvalue/document.h> #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/vespalib/stllike/allocator.h> #include <vespa/vespalib/stllike/hash_map.h> @@ -15,6 +14,12 @@ namespace search::index { class Schema; } +namespace document { + class FieldValue; + class StringFieldValue; + class ArrayFieldValue; + class WeightedSetFieldValue; +} namespace search::memoryindex { class IOrderedFieldIndexInserter; @@ -299,7 +304,7 @@ public: /** * Invert a normal text field, based on annotations. */ - void invertField(uint32_t docId, const document::FieldValue::UP &val); + void invertField(uint32_t docId, const std::unique_ptr<document::FieldValue> &val); /** * Setup remove of word in old version of document. diff --git a/searchlib/src/vespa/searchlib/memoryindex/invert_task.cpp b/searchlib/src/vespa/searchlib/memoryindex/invert_task.cpp index fb6e1328b8b..8fa9de7da74 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/invert_task.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/invert_task.cpp @@ -5,6 +5,7 @@ #include "field_inverter.h" #include "invert_context.h" #include "url_field_inverter.h" +#include <vespa/document/fieldvalue/document.h> namespace search::memoryindex { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp index 597cfe8eb40..487fb9e329f 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp @@ -322,7 +322,7 @@ public: if (_field_value.get()) { return std::move(_field_value); } - return std::make_unique<StringFieldValue>(_str.str()); + return StringFieldValue::make(_str.str()); } }; diff --git a/staging_vespalib/src/tests/state_server/state_server_test.cpp b/staging_vespalib/src/tests/state_server/state_server_test.cpp index 5903970b740..cb80a44a675 100644 --- a/staging_vespalib/src/tests/state_server/state_server_test.cpp +++ b/staging_vespalib/src/tests/state_server/state_server_test.cpp @@ -12,7 +12,7 @@ #include <vespa/vespalib/net/simple_metrics_producer.h> #include <vespa/vespalib/net/simple_component_config_producer.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/util/child_process.h> +#include <vespa/vespalib/process/process.h> #include <vespa/vespalib/net/state_explorer.h> #include <vespa/vespalib/net/slime_explorer.h> #include <vespa/vespalib/net/generic_state_handler.h> @@ -39,8 +39,8 @@ std::map<vespalib::string,vespalib::string> empty_params; //----------------------------------------------------------------------------- vespalib::string run_cmd(const vespalib::string &cmd) { - std::string out; - ASSERT_TRUE(ChildProcess::run(cmd.c_str(), out)); + vespalib::string out; + ASSERT_TRUE(Process::run(cmd.c_str(), out)); return out; } @@ -511,7 +511,7 @@ TEST("require that generic state can be explored") { check_json(json_list_two, state_handler.get(host_tag, root_path + "list/two", empty_params)); } -TEST_MAIN_WITH_PROCESS_PROXY() { +TEST_MAIN() { mkdir("var", S_IRWXU); mkdir("var/run", S_IRWXU); TEST_RUN_ALL(); diff --git a/storage/src/tests/storageserver/documentapiconvertertest.cpp b/storage/src/tests/storageserver/documentapiconvertertest.cpp index 4b3cd3dfe17..c375443b265 100644 --- a/storage/src/tests/storageserver/documentapiconvertertest.cpp +++ b/storage/src/tests/storageserver/documentapiconvertertest.cpp @@ -8,6 +8,7 @@ #include <vespa/document/select/parser.h> #include <vespa/document/test/make_document_bucket.h> #include <vespa/document/update/documentupdate.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/documentapi/documentapi.h> #include <vespa/messagebus/emptyreply.h> #include <vespa/messagebus/error.h> diff --git a/storage/src/vespa/storage/persistence/CMakeLists.txt b/storage/src/vespa/storage/persistence/CMakeLists.txt index c737d2bed28..7715517a791 100644 --- a/storage/src/vespa/storage/persistence/CMakeLists.txt +++ b/storage/src/vespa/storage/persistence/CMakeLists.txt @@ -18,6 +18,5 @@ vespa_add_library(storage_spersistence OBJECT splitbitdetector.cpp splitjoinhandler.cpp testandsethelper.cpp - types.cpp DEPENDS ) diff --git a/storage/src/vespa/storage/persistence/asynchandler.h b/storage/src/vespa/storage/persistence/asynchandler.h index 71c54e99f75..d50ebf36c81 100644 --- a/storage/src/vespa/storage/persistence/asynchandler.h +++ b/storage/src/vespa/storage/persistence/asynchandler.h @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "types.h" #include "messages.h" #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/removelocation.h> @@ -16,12 +15,14 @@ namespace spi { } class PersistenceUtil; class BucketOwnershipNotifier; +class MessageTracker; /** * Handle async operations that uses a sequenced executor. * It is stateless and thread safe. */ -class AsyncHandler : public Types { +class AsyncHandler { + using MessageTrackerUP = std::unique_ptr<MessageTracker>; public: AsyncHandler(const PersistenceUtil&, spi::PersistenceProvider&, BucketOwnershipNotifier &, vespalib::ISequencedTaskExecutor & executor, const document::BucketIdFactory & bucketIdFactory); diff --git a/storage/src/vespa/storage/persistence/merge_bucket_info_syncer.h b/storage/src/vespa/storage/persistence/merge_bucket_info_syncer.h index b3386c591e6..cd7e9690316 100644 --- a/storage/src/vespa/storage/persistence/merge_bucket_info_syncer.h +++ b/storage/src/vespa/storage/persistence/merge_bucket_info_syncer.h @@ -2,6 +2,8 @@ #pragma once +#include <memory> + namespace storage::spi { class Bucket; } namespace storage { diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index 8287fe27509..73361c7e8d6 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -10,6 +10,7 @@ #include <vespa/persistence/spi/docentry.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/document/fieldset/fieldsets.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/isequencedtaskexecutor.h> @@ -239,7 +240,7 @@ MergeHandler::buildBucketInfoList( for (const auto& entry : entries) { api::GetBucketDiffCommand::Entry diff; - diff._gid = GlobalId(); + diff._gid = document::GlobalId(); // We do not know doc sizes at this point, so just set to 0 diff._headerSize = 0; diff._bodySize = 0; @@ -449,7 +450,7 @@ MergeHandler::fetchLocalData( api::ApplyBucketDiffCommand::Entry& e(*iter); if (!docEntry.isRemove()) { - const Document* doc = docEntry.getDocument(); + const document::Document* doc = docEntry.getDocument(); assert(doc != nullptr); assertContainedInBucket(doc->getId(), bucket, idFactory); e._docName = doc->getId().toString(); @@ -459,7 +460,7 @@ MergeHandler::fetchLocalData( memcpy(&e._headerBlob[0], stream.peek(), stream.size()); e._bodyBlob.clear(); } else { - const DocumentId* docId = docEntry.getDocumentId(); + const document::DocumentId* docId = docEntry.getDocumentId(); assert(docId != nullptr); assertContainedInBucket(*docId, bucket, idFactory); if (e._entry._flags & DELETED) { @@ -495,7 +496,7 @@ MergeHandler::deserializeDiffDocument( const api::ApplyBucketDiffCommand::Entry& e, const document::DocumentTypeRepo& repo) const { - auto doc = std::make_unique<Document>(); + auto doc = std::make_unique<document::Document>(); vespalib::nbostream hbuf(&e._headerBlob[0], e._headerBlob.size()); if (e._bodyBlob.size() > 0) { // TODO Remove this branch and add warning on error. @@ -519,8 +520,8 @@ MergeHandler::applyDiffEntry(std::shared_ptr<ApplyBucketDiffState> async_results spi::Timestamp timestamp(e._entry._timestamp); if (!(e._entry._flags & (DELETED | DELETED_IN_PLACE))) { // Regular put entry - Document::SP doc(deserializeDiffDocument(e, repo)); - DocumentId docId = doc->getId(); + std::shared_ptr<document::Document> doc(deserializeDiffDocument(e, repo)); + document::DocumentId docId = doc->getId(); auto complete = std::make_unique<ApplyBucketDiffEntryComplete>(std::move(async_results), std::move(docId), std::move(throttle_token), "put", _clock, _env._metrics.merge_handler_metrics.put_latency); @@ -557,7 +558,7 @@ MergeHandler::applyDiffLocally( async_results->mark_stale_bucket_info(); DocEntryList entries; - populateMetaData(bucket, MAX_TIMESTAMP, entries, context); + populateMetaData(bucket, Timestamp::max(), entries, context); const document::DocumentTypeRepo & repo = _env.getDocumentTypeRepo(); diff --git a/storage/src/vespa/storage/persistence/mergehandler.h b/storage/src/vespa/storage/persistence/mergehandler.h index 1ed2fa878bc..3f631acbef5 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.h +++ b/storage/src/vespa/storage/persistence/mergehandler.h @@ -13,19 +13,17 @@ */ #pragma once -#include "types.h" #include "merge_bucket_info_syncer.h" #include <vespa/persistence/spi/bucket.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storage/common/cluster_context.h> #include <vespa/storage/common/messagesender.h> #include <vespa/vespalib/util/monitored_refcount.h> +#include <vespa/storageframework/generic/clock/time.h> #include <atomic> -namespace vespalib { -class ISequencedTaskExecutor; -} - +namespace vespalib { class ISequencedTaskExecutor; } +namespace document { class Document; } namespace storage { namespace spi { @@ -36,10 +34,12 @@ namespace spi { class PersistenceUtil; class ApplyBucketDiffState; class MergeStatus; +class MessageTracker; -class MergeHandler : public Types, - public MergeBucketInfoSyncer { - +class MergeHandler : public MergeBucketInfoSyncer { +private: + using MessageTrackerUP = std::unique_ptr<MessageTracker>; + using Timestamp = framework::MicroSecTime; public: enum StateFlag { IN_USE = 0x01, @@ -92,7 +92,7 @@ public: private: using DocEntryList = std::vector<std::unique_ptr<spi::DocEntry>>; const framework::Clock &_clock; - const ClusterContext &_cluster_context; + const ClusterContext &_cluster_context; PersistenceUtil &_env; spi::PersistenceProvider &_spi; std::unique_ptr<vespalib::MonitoredRefCount> _monitored_ref_count; @@ -129,9 +129,8 @@ private: DocEntryList & entries, spi::Context& context) const; - Document::UP deserializeDiffDocument( - const api::ApplyBucketDiffCommand::Entry& e, - const document::DocumentTypeRepo& repo) const; + std::unique_ptr<document::Document> + deserializeDiffDocument(const api::ApplyBucketDiffCommand::Entry& e, const document::DocumentTypeRepo& repo) const; }; } // storage diff --git a/storage/src/vespa/storage/persistence/persistenceutil.cpp b/storage/src/vespa/storage/persistence/persistenceutil.cpp index 2781cc61b83..77e8532f0d2 100644 --- a/storage/src/vespa/storage/persistence/persistenceutil.cpp +++ b/storage/src/vespa/storage/persistence/persistenceutil.cpp @@ -3,6 +3,7 @@ #include "persistenceutil.h" #include <vespa/persistence/spi/persistenceprovider.h> #include <vespa/storageapi/messageapi/bucketinforeply.h> +#include <vespa/document/base/documentid.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/log/bufferedlogger.h> diff --git a/storage/src/vespa/storage/persistence/persistenceutil.h b/storage/src/vespa/storage/persistence/persistenceutil.h index 4130a276239..c3fcb68ddc8 100644 --- a/storage/src/vespa/storage/persistence/persistenceutil.h +++ b/storage/src/vespa/storage/persistence/persistenceutil.h @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "types.h" #include <vespa/storage/common/servicelayercomponent.h> #include <vespa/storage/persistence/filestorage/filestorhandler.h> #include <vespa/storage/persistence/filestorage/filestormetrics.h> +#include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storageframework/generic/clock/timer.h> #include <vespa/storageapi/messageapi/returncode.h> #include <vespa/persistence/spi/result.h> @@ -15,6 +15,7 @@ namespace storage::api { class StorageMessage; class StorageReply; + class BucketInfo; } namespace storage::spi { @@ -25,7 +26,7 @@ namespace storage { class PersistenceUtil; -class MessageTracker : protected Types { +class MessageTracker { public: using UP = std::unique_ptr<MessageTracker>; @@ -48,7 +49,7 @@ public: } /** Utility function to be able to write a bit less in client. */ - void fail(uint32_t result, const String& message = "") { + void fail(uint32_t result, const vespalib::string & message = "") { fail(api::ReturnCode((api::ReturnCode::Result)result, message)); } /** Set the request to fail with the given failure. */ diff --git a/storage/src/vespa/storage/persistence/processallhandler.cpp b/storage/src/vespa/storage/persistence/processallhandler.cpp index 6d6723a0185..c22b08c5ca5 100644 --- a/storage/src/vespa/storage/persistence/processallhandler.cpp +++ b/storage/src/vespa/storage/persistence/processallhandler.cpp @@ -4,6 +4,7 @@ #include "bucketprocessor.h" #include "persistenceutil.h" #include <vespa/document/fieldset/fieldsets.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/persistence/spi/persistenceprovider.h> #include <vespa/persistence/spi/docentry.h> #include <vespa/vespalib/util/stringfmt.h> diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp index b7344098698..6d15cc06cdf 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "provider_error_wrapper.h" -#include "persistenceutil.h" #include <vespa/persistence/spi/docentry.h> #include <vespa/vespalib/util/idestructorcallback.h> diff --git a/storage/src/vespa/storage/persistence/types.cpp b/storage/src/vespa/storage/persistence/types.cpp deleted file mode 100644 index 3bf116bb2ee..00000000000 --- a/storage/src/vespa/storage/persistence/types.cpp +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "types.h" - -namespace storage { - -const framework::MicroSecTime Types::MAX_TIMESTAMP(framework::MicroSecTime::max()); -const framework::MicroSecTime Types::UNSET_TIMESTAMP(0); - -} // storage diff --git a/storage/src/vespa/storage/persistence/types.h b/storage/src/vespa/storage/persistence/types.h index 8dff11468a0..da004f85075 100644 --- a/storage/src/vespa/storage/persistence/types.h +++ b/storage/src/vespa/storage/persistence/types.h @@ -4,7 +4,6 @@ #include <iosfwd> #include <vespa/document/bucket/bucketid.h> #include <vespa/document/base/documentid.h> -#include <vespa/document/fieldvalue/document.h> #include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storageapi/buckets/bucketinfo.h> #include <vespa/storageapi/defs.h> @@ -17,18 +16,12 @@ class MessageTracker; struct Types { typedef document::BucketId BucketId; - typedef document::Document Document; typedef document::DocumentId DocumentId; typedef document::GlobalId GlobalId; typedef framework::MicroSecTime Timestamp; - typedef Timestamp RevertToken; typedef vespalib::string String; typedef api::BucketInfo BucketInfo; using MessageTrackerUP = std::unique_ptr<MessageTracker>; - - static const framework::MicroSecTime MAX_TIMESTAMP; - static const framework::MicroSecTime UNSET_TIMESTAMP; - protected: ~Types() {} // Noone should refer to objects as Types objects }; diff --git a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp index d95349fc88b..c465f974a9b 100644 --- a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp +++ b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp @@ -5,6 +5,7 @@ #include <vespa/document/bucket/bucketidfactory.h> #include <vespa/document/update/documentupdate.h> #include <vespa/documentapi/documentapi.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/storage/common/bucket_resolver.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storageapi/message/documentsummary.h> diff --git a/storage/src/vespa/storage/visiting/recoveryvisitor.cpp b/storage/src/vespa/storage/visiting/recoveryvisitor.cpp index 1c3c97d7dcb..9d1b2c83266 100644 --- a/storage/src/vespa/storage/visiting/recoveryvisitor.cpp +++ b/storage/src/vespa/storage/visiting/recoveryvisitor.cpp @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - #include "recoveryvisitor.h" #include <vespa/persistence/spi/docentry.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/documentapi/messagebus/messages/visitor.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/vespalib/stllike/hash_map.hpp> diff --git a/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp b/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp index 0d5ee1e507c..3dc01725a3e 100644 --- a/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp +++ b/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> #include <vbench/test/all.h> -#include <vespa/vespalib/util/child_process.h> +#include <vespa/vespalib/process/process.h> #include <vespa/vespalib/net/crypto_engine.h> using namespace vbench; -using vespalib::ChildProcess; +using vespalib::Process; using InputReader = vespalib::InputReader; using OutputWriter = vespalib::OutputWriter; @@ -31,8 +31,8 @@ void readUntil(Input &input, SimpleBuffer &buffer, const string &end) { } TEST("dumpurl usage") { - std::string out; - EXPECT_FALSE(ChildProcess::run("../../apps/dumpurl/vbench_dumpurl_app", out)); + vespalib::string out; + EXPECT_FALSE(Process::run("../../apps/dumpurl/vbench_dumpurl_app", out)); fprintf(stderr, "%s\n", out.c_str()); } @@ -47,11 +47,11 @@ TEST_MT_F("run dumpurl", 2, ServerSocket()) { out.write("\r\n"); out.write("data"); } else { - std::string out; - EXPECT_TRUE(ChildProcess::run(strfmt("../../apps/dumpurl/vbench_dumpurl_app localhost %d /foo", - f1.port()).c_str(), out)); + vespalib::string out; + EXPECT_TRUE(Process::run(strfmt("../../apps/dumpurl/vbench_dumpurl_app localhost %d /foo", + f1.port()).c_str(), out)); fprintf(stderr, "%s\n", out.c_str()); } } -TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/app_vbench/app_vbench_test.cpp b/vbench/src/tests/app_vbench/app_vbench_test.cpp index f48c6fe0b05..f3223e84092 100644 --- a/vbench/src/tests/app_vbench/app_vbench_test.cpp +++ b/vbench/src/tests/app_vbench/app_vbench_test.cpp @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> #include <vbench/test/all.h> -#include <vespa/vespalib/util/child_process.h> +#include <vespa/vespalib/process/process.h> #include <vespa/vespalib/net/crypto_engine.h> #include <vespa/vespalib/net/tls/tls_crypto_engine.h> #include <vespa/vespalib/test/make_tls_options_for_testing.h> @@ -11,7 +11,7 @@ #include <fcntl.h> using namespace vbench; -using vespalib::ChildProcess; +using vespalib::Process; using InputReader = vespalib::InputReader; using OutputWriter = vespalib::OutputWriter; @@ -31,8 +31,8 @@ void write_file(const vespalib::string &file_name, const vespalib::string &conte } TEST("vbench usage") { - std::string out; - EXPECT_FALSE(ChildProcess::run("../../apps/vbench/vbench_app", out)); + vespalib::string out; + EXPECT_FALSE(Process::run("../../apps/vbench/vbench_app", out)); fprintf(stderr, "%s\n", out.c_str()); } @@ -68,18 +68,18 @@ struct Servers { TEST_MT_F("run vbench", 2, Servers()) { if (thread_id == 0) { - std::string out; - EXPECT_TRUE(ChildProcess::run(strfmt("sed 's/_LOCAL_PORT_/%d/' vbench.cfg.template > vbench.cfg", f1.portal->listen_port()).c_str())); - EXPECT_TRUE(ChildProcess::run("../../apps/vbench/vbench_app run vbench.cfg 2> vbench.out", out)); + vespalib::string out; + EXPECT_TRUE(Process::run(strfmt("sed 's/_LOCAL_PORT_/%d/' vbench.cfg.template > vbench.cfg", f1.portal->listen_port()).c_str())); + EXPECT_TRUE(Process::run("../../apps/vbench/vbench_app run vbench.cfg 2> vbench.out", out)); fprintf(stderr, "null crypto: %s\n", out.c_str()); EXPECT_GREATER(f1.my_get.cnt, 10u); } else { - std::string tls_out; - EXPECT_TRUE(ChildProcess::run(strfmt("sed 's/_LOCAL_PORT_/%d/' vbench.tls.cfg.template > vbench.tls.cfg", f1.tls_portal->listen_port()).c_str())); - EXPECT_TRUE(ChildProcess::run("../../apps/vbench/vbench_app run vbench.tls.cfg 2> vbench.tls.out", tls_out)); + vespalib::string tls_out; + EXPECT_TRUE(Process::run(strfmt("sed 's/_LOCAL_PORT_/%d/' vbench.tls.cfg.template > vbench.tls.cfg", f1.tls_portal->listen_port()).c_str())); + EXPECT_TRUE(Process::run("../../apps/vbench/vbench_app run vbench.tls.cfg 2> vbench.tls.out", tls_out)); fprintf(stderr, "tls crypto: %s\n", tls_out.c_str()); EXPECT_GREATER(f1.my_tls_get.cnt, 10u); } } -TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzRoleInformation.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzRoleInformation.java new file mode 100644 index 00000000000..dcc3452a69a --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzRoleInformation.java @@ -0,0 +1,121 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.api; + +import com.yahoo.vespa.athenz.client.zms.bindings.RoleEntity; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * @author olaa + */ +public class AthenzRoleInformation extends AthenzRole { + + private final boolean isSelfServe; + private final boolean reviewEnabled; + private final Optional<MembershipRequest> pendingRequest; + private final List<AuditLogEntry> auditLog; + + public AthenzRoleInformation(AthenzDomain domain, String roleName, boolean isSelfServe, boolean reviewEnabled, Optional<MembershipRequest> pendingRequest, List<AuditLogEntry> auditLog) { + super(domain, roleName); + this.isSelfServe = isSelfServe; + this.reviewEnabled = reviewEnabled; + this.pendingRequest = pendingRequest; + this.auditLog = auditLog; + } + + public boolean isSelfServe() { + return isSelfServe; + } + + public boolean isReviewEnabled() { + return reviewEnabled; + } + + public Optional<MembershipRequest> getPendingRequest() { + return pendingRequest; + } + + public List<AuditLogEntry> getAuditLog() { + return auditLog; + } + + public static AthenzRoleInformation fromRoleEntity(RoleEntity roleEntity) { + var role = fromResourceNameString(roleEntity.roleName()); + var isSelfServe = roleEntity.selfServe() != null && roleEntity.selfServe(); + var reviewEnabled = roleEntity.reviewEnabled() != null && roleEntity.reviewEnabled(); + var pendingRequest = roleEntity.roleMembers() + .stream() + .filter(member -> member.pendingApproval()) + .map(member -> new MembershipRequest(member.memberName(), member.auditRef(), member.requestTime(), member.active())) + .findFirst(); + var auditLog = roleEntity.auditLog() + .stream() + .map(entry -> new AuditLogEntry(entry.getAdmin(), entry.getAction(), entry.getAuditRef(), entry.getCreated())) + .collect(Collectors.toList()); + return new AthenzRoleInformation(role.domain(), role.roleName(), isSelfServe, reviewEnabled, pendingRequest, auditLog); + } + + + public static class MembershipRequest { + private final String memberName; + private final String reason; + private final String creationTime; + private final boolean active; + + public MembershipRequest(String memberName, String reason, String creationTime, boolean active) { + this.memberName = memberName; + this.reason = reason; + this.creationTime = creationTime; + this.active = active; + } + + public String getMemberName() { + return memberName; + } + + public String getReason() { + return reason; + } + + public String getCreationTime() { + return creationTime; + } + + public boolean isActive() { + return active; + } + } + + public static class AuditLogEntry { + private final String approver; + private final String action; + private final String reason; + private final String creationTime; + + public AuditLogEntry(String approver, String action, String reason, String creationTime) { + this.approver = approver; + this.action = action; + this.reason = reason; + this.creationTime = creationTime; + } + + public String getApprover() { + return approver; + } + + public String getAction() { + return action; + } + + public String getReason() { + return reason; + } + + public String getCreationTime() { + return creationTime; + } + } + +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java index 32f54255262..eef833c91a7 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPolicy; import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; +import com.yahoo.vespa.athenz.api.AthenzRoleInformation; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.OAuthCredentials; import com.yahoo.vespa.athenz.client.ErrorHandler; @@ -301,10 +302,10 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { } @Override - public void approvePendingRoleMembership(AthenzRole athenzRole, AthenzIdentity athenzIdentity, Instant expiry, - Optional<String> reason, Optional<OAuthCredentials> oAuthCredentials) { + public void decidePendingRoleMembership(AthenzRole athenzRole, AthenzIdentity athenzIdentity, Instant expiry, + Optional<String> reason, Optional<OAuthCredentials> oAuthCredentials, boolean approve) { URI uri = zmsUrl.resolve(String.format("domain/%s/role/%s/member/%s/decision", athenzRole.domain().getName(), athenzRole.roleName(), athenzIdentity.getFullName())); - MembershipEntity membership = new MembershipEntity.RoleMembershipEntity(athenzIdentity.getFullName(), true, athenzRole.roleName(), Long.toString(expiry.getEpochSecond())); + MembershipEntity membership = new MembershipEntity.RoleMembershipEntity(athenzIdentity.getFullName(), approve, athenzRole.roleName(), Long.toString(expiry.getEpochSecond())); var requestBuilder = RequestBuilder.put() .setUri(uri) @@ -406,6 +407,13 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { execute(request, response -> readEntity(response, Void.class)); } + public AthenzRoleInformation getFullRoleInformation(AthenzRole role) { + var uri = zmsUrl.resolve(String.format("domain/%s/role/%s?pending=true&auditLog=true", role.domain().getName(), role.roleName())); + var request = RequestBuilder.get(uri).build(); + var roleEntity = execute(request, response -> readEntity(response, RoleEntity.class)); + return AthenzRoleInformation.fromRoleEntity(roleEntity); + } + private static Header createCookieHeader(OAuthCredentials oAuthCredentials) { return new BasicHeader("Cookie", oAuthCredentials.asCookie()); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java index 95b7d9b8976..3ff2ff843a0 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPolicy; import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; +import com.yahoo.vespa.athenz.api.AthenzRoleInformation; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.OAuthCredentials; @@ -59,8 +60,8 @@ public interface ZmsClient extends AutoCloseable { Map<AthenzIdentity, String> listPendingRoleApprovals(AthenzRole athenzRole); - void approvePendingRoleMembership(AthenzRole athenzRole, AthenzIdentity athenzIdentity, Instant expiry, - Optional<String> reason, Optional<OAuthCredentials> oAuthCredentials); + void decidePendingRoleMembership(AthenzRole athenzRole, AthenzIdentity athenzIdentity, Instant expiry, + Optional<String> reason, Optional<OAuthCredentials> oAuthCredentials, boolean approve); List<AthenzIdentity> listMembers(AthenzRole athenzRole); @@ -80,5 +81,7 @@ public interface ZmsClient extends AutoCloseable { void createSubdomain(AthenzDomain parent, String name); + AthenzRoleInformation getFullRoleInformation(AthenzRole role); + void close(); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/RoleEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/RoleEntity.java index 28b1f5d3206..3ee0c717f19 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/RoleEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/RoleEntity.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.ArrayList; import java.util.List; /** @@ -15,11 +16,21 @@ import java.util.List; public class RoleEntity { private final String roleName; private final List<Member> roleMembers; + private final Boolean selfServe; + private final Boolean reviewEnabled; + private final List<AuditLogEntry> auditLog; @JsonCreator - public RoleEntity(@JsonProperty("roleName") String roleName, @JsonProperty("roleMembers") List<Member> roleMembers) { + public RoleEntity(@JsonProperty("roleName") String roleName, + @JsonProperty("roleMembers") List<Member> roleMembers, + @JsonProperty("selfServe") Boolean selfServe, + @JsonProperty("reviewEnabled") Boolean reviewEnabled, + @JsonProperty("auditLog") List<AuditLogEntry> auditLog) { this.roleName = roleName; this.roleMembers = roleMembers; + this.selfServe = selfServe; + this.reviewEnabled = reviewEnabled; + this.auditLog = auditLog == null ? new ArrayList<>() : auditLog; } public String roleName() { @@ -30,19 +41,37 @@ public class RoleEntity { return roleMembers; } + public Boolean selfServe() { + return selfServe; + } + + public Boolean reviewEnabled() { + return reviewEnabled; + } + + public List<AuditLogEntry> auditLog() { + return auditLog; + } + @JsonIgnoreProperties(ignoreUnknown = true) public static final class Member { private final String memberName; private final boolean active; private final boolean approved; private final String auditRef; + private final String requestTime; @JsonCreator - public Member(@JsonProperty("memberName") String memberName, @JsonProperty("active") boolean active, @JsonProperty("approved") boolean approved, @JsonProperty("auditRef") String auditRef) { + public Member(@JsonProperty("memberName") String memberName, + @JsonProperty("active") boolean active, + @JsonProperty("approved") boolean approved, + @JsonProperty("auditRef") String auditRef, + @JsonProperty("requestTime") String requestTime) { this.memberName = memberName; this.active = active; this.approved = approved; this.auditRef = auditRef; + this.requestTime = requestTime; } public String memberName() { @@ -56,5 +85,55 @@ public class RoleEntity { public String auditRef() { return auditRef; } + + public String requestTime() { + return requestTime; + } + + public boolean active() { + return active; + } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static final class AuditLogEntry { + private final String member; + private final String admin; + private final String action; + private final String auditRef; + private final String created; + + @JsonCreator + public AuditLogEntry(@JsonProperty("member") String member, + @JsonProperty("admin") String admin, + @JsonProperty("created") String created, + @JsonProperty("action") String action, + @JsonProperty("auditRef") String auditRef) { + this.member = member; + this.admin = admin; + this.created = created; + this.action = action; + this.auditRef = auditRef; + } + + public String getMember() { + return member; + } + + public String getAdmin() { + return admin; + } + + public String getAction() { + return action; + } + + public String getAuditRef() { + return auditRef; + } + + public String getCreated() { + return created; + } } } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java index 2a774995601..f99274d3f2b 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java @@ -7,7 +7,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.container.jdisc.messagebus.SessionCache; -import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.document.DocumentTypeManager; import com.yahoo.documentapi.metrics.DocumentApiMetrics; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Request; @@ -45,12 +45,12 @@ public class FeedHandler extends ThreadedHttpRequestHandler { @Inject public FeedHandler(ContainerThreadPool threadpool, Metric metric, - DocumentmanagerConfig documentManagerConfig, + DocumentTypeManager documentTypeManager, SessionCache sessionCache, MetricReceiver metricReceiver) { super(threadpool.executor(), metric); metricsHelper = new DocumentApiMetrics(metricReceiver, "vespa.http.server"); - feedHandlerV3 = new FeedHandlerV3(threadpool.executor(), metric, documentManagerConfig, sessionCache, metricsHelper); + feedHandlerV3 = new FeedHandlerV3(threadpool.executor(), metric, documentTypeManager, sessionCache, metricsHelper); feedReplyHandler = new FeedReplyReader(metric, metricsHelper); } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java index 95e98f325e1..c8828df6d54 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java @@ -7,7 +7,6 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.container.jdisc.messagebus.SessionCache; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.metrics.DocumentApiMetrics; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.ReferencedResource; @@ -45,11 +44,11 @@ public class FeedHandlerV3 extends ThreadedHttpRequestHandler { public FeedHandlerV3(Executor executor, Metric metric, - DocumentmanagerConfig documentManagerConfig, + DocumentTypeManager documentTypeManager, SessionCache sessionCache, DocumentApiMetrics metricsHelper) { super(executor, metric); - docTypeManager = new DocumentTypeManager(documentManagerConfig); + docTypeManager = documentTypeManager; this.sessionCache = sessionCache; feedReplyHandler = new FeedReplyReader(metric, metricsHelper); cron = new ScheduledThreadPoolExecutor(1, ThreadFactoryFactory.getThreadFactory("feed-handler-v3-janitor")); diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerTest.java index 00df7c8b6fa..f3ea8fb5a80 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.http.server; import com.yahoo.container.handler.threadpool.ContainerThreadPool; import com.yahoo.container.jdisc.RequestHandlerTestDriver; +import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.jdisc.handler.OverloadException; import com.yahoo.metrics.simple.MetricReceiver; @@ -24,7 +25,7 @@ public class FeedHandlerTest { FeedHandler handler = new FeedHandler( new RejectingContainerThreadpool(), new CollectingMetric(), - new DocumentmanagerConfig(new DocumentmanagerConfig.Builder().enablecompression(true)), + new DocumentTypeManager(new DocumentmanagerConfig.Builder().enablecompression(true).build()), null /* session cache */, MetricReceiver.nullImplementation); var responseHandler = new RequestHandlerTestDriver.MockResponseHandler(); diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java index 2f4afb0c2a5..a5a8f4cb5bd 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java @@ -112,7 +112,7 @@ public class FeedHandlerV3Test { } private FeedHandlerV3 setupFeederHandler(Executor threadPool) { - DocumentmanagerConfig docMan = new DocumentmanagerConfig(new DocumentmanagerConfig.Builder().enablecompression(true)); + DocumentTypeManager docMan = new DocumentTypeManager(new DocumentmanagerConfig.Builder().enablecompression(true).build()); FeedHandlerV3 feedHandlerV3 = new FeedHandlerV3( threadPool, metric, diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 80308a75bd5..6fb832b21d3 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -29,7 +29,6 @@ vespa_define_module( src/tests/btree/btree_store src/tests/btree/btree-scan-speed src/tests/btree/btree-stress - src/tests/child_process src/tests/component src/tests/compress src/tests/compression diff --git a/vespalib/src/tests/assert/assert_test.cpp b/vespalib/src/tests/assert/assert_test.cpp index 49f4d2194f4..e953fc439c3 100644 --- a/vespalib/src/tests/assert/assert_test.cpp +++ b/vespalib/src/tests/assert/assert_test.cpp @@ -1,6 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/child_process.h> +#include <vespa/vespalib/process/process.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/assert.h> #include <vespa/vespalib/io/fileutil.h> @@ -16,24 +16,21 @@ TEST("that it borks the first time.") { vespalib::rmdir("var", true); ASSERT_TRUE(vespalib::mkdir(assertDir, true)); { - ChildProcess proc("ulimit -c 0 && exec env VESPA_HOME=./ ./vespalib_asserter_app myassert 10000"); - proc.wait(); - ASSERT_EQUAL(proc.getExitCode() & 0x7f, 6); + Process proc("ulimit -c 0 && exec env VESPA_HOME=./ ./vespalib_asserter_app myassert 10000"); + ASSERT_EQUAL(proc.join() & 0x7f, 6); } { - ChildProcess proc("ulimit -c 0 && exec env VESPA_HOME=./ ./vespalib_asserter_app myassert 10000"); - proc.readLine(assertName); - proc.wait(); - ASSERT_EQUAL(proc.getExitCode() & 0x7f, 0); + Process proc("ulimit -c 0 && exec env VESPA_HOME=./ ./vespalib_asserter_app myassert 10000"); + assertName = proc.read_line(); + ASSERT_EQUAL(proc.join() & 0x7f, 0); } ASSERT_EQUAL(0, unlink(assertName.c_str())); { - ChildProcess proc("ulimit -c 0 && exec env VESPA_HOME=./ ./vespalib_asserter_app myassert 10000"); - proc.wait(); - ASSERT_EQUAL(proc.getExitCode() & 0x7f, 6); + Process proc("ulimit -c 0 && exec env VESPA_HOME=./ ./vespalib_asserter_app myassert 10000"); + ASSERT_EQUAL(proc.join() & 0x7f, 6); } ASSERT_EQUAL(0, unlink(assertName.c_str())); ASSERT_TRUE(vespalib::rmdir("var", true)); } -TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp b/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp index 7d11990577d..4716e91c2c4 100644 --- a/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp +++ b/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp @@ -196,9 +196,11 @@ protected: using MyTree = typename Params::MyTree; using MyTreeIterator = typename MyTree::Iterator; using MyTreeConstIterator = typename MyTree::ConstIterator; + using KeyStore = IntStore; + using ValueStore = IntStore; GenerationHandler _generationHandler; - IntStore _keys; - IntStore _values; + KeyStore _keys; + ValueStore _values; MyTree _tree; MyTreeIterator _writeItr; vespalib::ThreadStackExecutor _writer; // 1 write thread @@ -344,7 +346,7 @@ template <typename Params> void Fixture<Params>::compact_keys() { - if constexpr (_keys.is_indirect) { + if constexpr (KeyStore::is_indirect) { auto to_hold = _keys.start_compact(); EntryRefFilter filter(_keys.get_num_buffers(), _keys.get_offset_bits()); filter.add_buffers(to_hold); @@ -366,7 +368,7 @@ template <typename Params> void Fixture<Params>::compact_values() { - if constexpr (_values.is_indirect) { + if constexpr (ValueStore::is_indirect) { auto to_hold = _values.start_compact(); EntryRefFilter filter(_values.get_num_buffers(), _values.get_offset_bits()); filter.add_buffers(to_hold); @@ -391,12 +393,12 @@ Fixture<Params>::consider_compact(uint32_t idx) if (_compact_tree.consider(idx) && !_tree.getAllocator().getNodeStore().has_held_buffers()) { compact_tree(); } - if constexpr (_keys.is_indirect) { + if constexpr (KeyStore::is_indirect) { if (_compact_keys.consider(idx) && !_keys.has_held_buffers()) { compact_keys(); } } - if constexpr (_values.is_indirect) { + if constexpr (ValueStore::is_indirect) { if (_compact_values.consider(idx) && !_values.has_held_buffers()) { compact_values(); } diff --git a/vespalib/src/tests/child_process/.gitignore b/vespalib/src/tests/child_process/.gitignore deleted file mode 100644 index 7e094c772a6..00000000000 --- a/vespalib/src/tests/child_process/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -child_process_test -vespalib_child_process_test_app diff --git a/vespalib/src/tests/child_process/CMakeLists.txt b/vespalib/src/tests/child_process/CMakeLists.txt deleted file mode 100644 index b0503f7b08c..00000000000 --- a/vespalib/src/tests/child_process/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vespalib_child_process_test_app TEST - SOURCES - child_process_test.cpp - DEPENDS - vespalib -) -vespa_add_test(NAME vespalib_child_process_test_app COMMAND vespalib_child_process_test_app COST 30) diff --git a/vespalib/src/tests/child_process/child_process_test.cpp b/vespalib/src/tests/child_process/child_process_test.cpp deleted file mode 100644 index d8ed707c1bf..00000000000 --- a/vespalib/src/tests/child_process/child_process_test.cpp +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/child_process.h> - -using vespalib::ChildProcess; - -TEST("simple run, ignore output") { - EXPECT_TRUE(ChildProcess::run("echo foo")); -} - -TEST("simple run, ignore output, failure") { - EXPECT_TRUE(!ChildProcess::run("false")); -} - -TEST("simple run, ignore output, timeout") { - EXPECT_TRUE(!ChildProcess::run("exec sleep 60", 10)); -} - -TEST("simple run") { - std::string out; - EXPECT_TRUE(ChildProcess::run("/bin/echo -n foo", out)); - EXPECT_EQUAL(out, "foo"); -} - -TEST("simple run, strip single-line trailing newline") { - std::string out; - EXPECT_TRUE(ChildProcess::run("echo foo", out)); - EXPECT_EQUAL(out, "foo"); -} - -TEST("simple run, don't strip multi-line output") { - std::string out; - EXPECT_TRUE(ChildProcess::run("perl -e 'print \"foo\\n\\n\"'", out)); - EXPECT_EQUAL(out, "foo\n\n"); -} - -TEST("simple run with input") { - std::string in = "bar"; - std::string out; - EXPECT_TRUE(ChildProcess::run(in, "cat", out)); - EXPECT_EQUAL(out, "bar"); -} - -TEST("simple run with input, strip single-line trailing newline") { - std::string in = "bar\n"; - std::string out; - EXPECT_TRUE(ChildProcess::run(in, "cat", out)); - EXPECT_EQUAL(out, "bar"); -} - -TEST("simple run with input, don't strip multi-line output") { - std::string in = "bar\n\n"; - std::string out; - EXPECT_TRUE(ChildProcess::run(in, "cat", out)); - EXPECT_EQUAL("bar\n\n", out); -} - -TEST_MT("simple run, partial output due to timeout", 2) { - std::string out; - std::vector<size_t> timeouts({150, 300, 3000, 6000, 60000}); - const char *my_cmd = "exec perl -e '$| = 1; print \"foo\\\n\"; sleep(600); print \"bar\\\n\"'"; - for (size_t timeout: timeouts) { - fprintf(stderr, "... verifying partial output with%s input (timeout = %zu)\n", - (thread_id == 0) ? "out" : "", timeout); - if (thread_id == 0) { - out.clear(); - EXPECT_TRUE(!ChildProcess::run(my_cmd, out, timeout)); - } else { - out.clear(); - std::string in = "ignored\n"; - EXPECT_TRUE(!ChildProcess::run(in, my_cmd, out, timeout)); - } - if (out == "foo") { - break; - } - } - EXPECT_EQUAL(out, "foo"); -} - -TEST("proc failure") { - ChildProcess proc("false"); - // read with length 0 will wait for output - EXPECT_TRUE(proc.read(NULL, 0) == 0); - EXPECT_TRUE(proc.wait(60000)); - EXPECT_TRUE(!proc.running()); - EXPECT_TRUE(proc.failed()); -} - -TEST("basic read/write") { - int x; - int read; - char buf[64]; - ChildProcess proc("cat"); - - EXPECT_TRUE(proc.running()); - EXPECT_TRUE(!proc.failed()); - EXPECT_TRUE(proc.write("foo", 3)); - for (x = 0, read = 0; x < 10 && read < 3; ++x) { - read += proc.read(buf + read, sizeof(buf) - read); - } - EXPECT_TRUE(read == 3 && memcmp(buf, "foo", 3) == 0); - EXPECT_TRUE(proc.write("bar!", 4)); - for (x = 0, read = 0; x < 10 && read < 4; ++x) { - read += proc.read(buf + read, sizeof(buf) - read); - } - EXPECT_TRUE(read == 4 && memcmp(buf, "bar!", 4) == 0); - EXPECT_TRUE(!proc.eof()); // not eof yet - EXPECT_TRUE(proc.close()); // close stdin - EXPECT_TRUE(!proc.eof()); // eof not detected yet - EXPECT_TRUE(proc.read(buf, sizeof(buf)) == 0); - EXPECT_TRUE(proc.eof()); - EXPECT_TRUE(proc.read(buf, sizeof(buf)) == 0); - EXPECT_TRUE(proc.wait(60000)); - EXPECT_TRUE(!proc.running()); - EXPECT_TRUE(!proc.failed()); -} - -TEST("continuos run, readLine") { - std::string str; - ChildProcess proc("cat"); - - EXPECT_TRUE(proc.running()); - EXPECT_TRUE(!proc.failed()); - EXPECT_TRUE(proc.write("foo\n", 4)); - EXPECT_TRUE(proc.readLine(str)); - EXPECT_EQUAL(str, "foo"); - EXPECT_TRUE(proc.write("bar!\n", 5)); - EXPECT_TRUE(proc.readLine(str)); - EXPECT_EQUAL(str, "bar!"); - EXPECT_TRUE(!proc.eof()); // not eof yet - EXPECT_TRUE(proc.close()); // close stdin - EXPECT_TRUE(!proc.eof()); // eof not detected yet - EXPECT_TRUE(!proc.readLine(str)); - EXPECT_EQUAL(str, ""); - EXPECT_TRUE(proc.eof()); - EXPECT_TRUE(!proc.readLine(str)); - EXPECT_EQUAL(str, ""); - EXPECT_TRUE(proc.wait(60000)); - EXPECT_TRUE(!proc.running()); - EXPECT_TRUE(!proc.failed()); -} - -TEST("readLine, eof flushes last line") { - std::string str; - ChildProcess proc("cat"); - - EXPECT_TRUE(proc.running()); - EXPECT_TRUE(!proc.failed()); - EXPECT_TRUE(proc.write("foo\n", 4)); - EXPECT_TRUE(proc.readLine(str)); - EXPECT_EQUAL(str, "foo"); - EXPECT_TRUE(proc.write("bar!", 4)); - EXPECT_TRUE(!proc.eof()); // not eof yet - EXPECT_TRUE(proc.close()); // close stdin - EXPECT_TRUE(!proc.eof()); // eof not detected yet - EXPECT_TRUE(proc.readLine(str)); - EXPECT_EQUAL(str, "bar!"); - EXPECT_TRUE(proc.eof()); - EXPECT_TRUE(!proc.readLine(str)); - EXPECT_EQUAL(str, ""); - EXPECT_TRUE(proc.wait(60000)); - EXPECT_TRUE(!proc.running()); - EXPECT_TRUE(!proc.failed()); -} - -TEST("long continuos run, readLine") { - std::string in; - std::string out; - ChildProcess proc("cat"); - - EXPECT_TRUE(proc.running()); - EXPECT_TRUE(!proc.failed()); - for (uint32_t i = 0; i < 10000; ++i) { - char num[32]; - sprintf(num, "%d", i); - in.assign("long continous run, line "); - in.append(num).append("\n"); - EXPECT_TRUE(proc.write(in.data(), in.length())); - in.erase(in.size() - 1, 1); - EXPECT_TRUE(proc.readLine(out)); - EXPECT_EQUAL(in, out); - } - EXPECT_TRUE(proc.running()); - EXPECT_TRUE(!proc.failed()); -} - -TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/drop-file-from-cache/drop_file_from_cache_test.cpp b/vespalib/src/tests/drop-file-from-cache/drop_file_from_cache_test.cpp index e3344fb4106..63cec1caee1 100644 --- a/vespalib/src/tests/drop-file-from-cache/drop_file_from_cache_test.cpp +++ b/vespalib/src/tests/drop-file-from-cache/drop_file_from_cache_test.cpp @@ -1,25 +1,22 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/child_process.h> +#include <vespa/vespalib/process/process.h> -using vespalib::ChildProcess; +using vespalib::Process; TEST("no arguments") { - ChildProcess drop("../../apps/vespa-drop-file-from-cache/vespa-drop-file-from-cache"); - drop.wait(); - EXPECT_EQUAL(1, drop.getExitCode()); + Process drop("../../apps/vespa-drop-file-from-cache/vespa-drop-file-from-cache"); + EXPECT_EQUAL(1, drop.join()); } TEST("file does not exist") { - ChildProcess drop("../../apps/vespa-drop-file-from-cache/vespa-drop-file-from-cache not_exist"); - drop.wait(); - EXPECT_EQUAL(2, drop.getExitCode()); + Process drop("../../apps/vespa-drop-file-from-cache/vespa-drop-file-from-cache not_exist"); + EXPECT_EQUAL(2, drop.join()); } TEST("All is well") { - ChildProcess drop("../../apps/vespa-drop-file-from-cache/vespa-drop-file-from-cache vespalib_drop_file_from_cache_test_app"); - drop.wait(); - EXPECT_EQUAL(0, drop.getExitCode()); + Process drop("../../apps/vespa-drop-file-from-cache/vespa-drop-file-from-cache vespalib_drop_file_from_cache_test_app"); + EXPECT_EQUAL(0, drop.join()); } -TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp b/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp index 6c757b00af6..a15e9327d16 100644 --- a/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp +++ b/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exception.h> -#include <vespa/vespalib/util/child_process.h> +#include <vespa/vespalib/process/process.h> using namespace vespalib; @@ -14,27 +14,23 @@ using namespace vespalib; #endif TEST("that uncaught exception causes negative exitcode.") { - ChildProcess proc("ulimit -c 0 && exec ./vespalib_caught_uncaught_app uncaught"); - proc.wait(); - EXPECT_LESS(proc.getExitCode(), 0); + Process proc("ulimit -c 0 && exec ./vespalib_caught_uncaught_app uncaught"); + EXPECT_LESS(proc.join(), 0); } TEST("that uncaught silenced exception causes exitcode 66") { - ChildProcess proc("exec ./vespalib_caught_uncaught_app silenced_and_uncaught"); - proc.wait(); - EXPECT_EQUAL(proc.getExitCode(), 66); + Process proc("exec ./vespalib_caught_uncaught_app silenced_and_uncaught"); + EXPECT_EQUAL(proc.join(), 66); } TEST("that caught silenced exception followed by an uncaught causes negative exitcode.") { - ChildProcess proc("ulimit -c 0 && exec ./vespalib_caught_uncaught_app uncaught_after_silenced_and_caught"); - proc.wait(); - EXPECT_LESS(proc.getExitCode(), 0); + Process proc("ulimit -c 0 && exec ./vespalib_caught_uncaught_app uncaught_after_silenced_and_caught"); + EXPECT_LESS(proc.join(), 0); } TEST("that caught silenced exception causes exitcode 0") { - ChildProcess proc("exec ./vespalib_caught_uncaught_app silenced_and_caught"); - proc.wait(); - EXPECT_EQUAL(proc.getExitCode(), 0); + Process proc("exec ./vespalib_caught_uncaught_app silenced_and_caught"); + EXPECT_EQUAL(proc.join(), 0); } #ifndef __SANITIZE_ADDRESS__ @@ -42,23 +38,20 @@ TEST("that caught silenced exception causes exitcode 0") { // setrlimit with RLIMIT_AS is broken on Darwin #else TEST("that mmap within limits are fine cause exitcode 0") { - ChildProcess proc("exec ./vespalib_mmap_app 150000000 10485760 1"); - proc.wait(); - EXPECT_EQUAL(proc.getExitCode(), 0); + Process proc("exec ./vespalib_mmap_app 150000000 10485760 1"); + EXPECT_EQUAL(proc.join(), 0); } TEST("that mmap beyond limits cause negative exitcode.") { - ChildProcess proc("ulimit -c 0 && exec ./vespalib_mmap_app 100000000 10485760 10"); - proc.wait(); - EXPECT_LESS(proc.getExitCode(), 0); + Process proc("ulimit -c 0 && exec ./vespalib_mmap_app 100000000 10485760 10"); + EXPECT_LESS(proc.join(), 0); } TEST("that mmap beyond limits with set VESPA_SILENCE_CORE_ON_OOM cause exitcode 66.") { - ChildProcess proc("VESPA_SILENCE_CORE_ON_OOM=1 exec ./vespalib_mmap_app 100000000 10485760 10"); - proc.wait(); - EXPECT_EQUAL(proc.getExitCode(), 66); + Process proc("VESPA_SILENCE_CORE_ON_OOM=1 exec ./vespalib_mmap_app 100000000 10485760 10"); + EXPECT_EQUAL(proc.join(), 66); } #endif #endif -TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/host_name/host_name_test.cpp b/vespalib/src/tests/host_name/host_name_test.cpp index ca5aebe8c95..b68762cebb3 100644 --- a/vespalib/src/tests/host_name/host_name_test.cpp +++ b/vespalib/src/tests/host_name/host_name_test.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/host_name.h> -#include <vespa/vespalib/util/child_process.h> using namespace vespalib; @@ -9,4 +8,4 @@ TEST("require that host name can be obtained") { EXPECT_NOT_EQUAL("", HostName::get()); } -TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/make_fixture_macros/make_fixture_macros_test.cpp b/vespalib/src/tests/make_fixture_macros/make_fixture_macros_test.cpp index 0264a5aad0e..1cddddf7edc 100644 --- a/vespalib/src/tests/make_fixture_macros/make_fixture_macros_test.cpp +++ b/vespalib/src/tests/make_fixture_macros/make_fixture_macros_test.cpp @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/child_process.h> +#include <vespa/vespalib/process/process.h> using namespace vespalib; -bool runPrint(const char *cmd) { - std::string out; - bool res = ChildProcess::run(cmd, out); +bool runPrint(const vespalib::string &cmd) { + vespalib::string out; + bool res = Process::run(cmd, out); fprintf(stderr, "%s", out.c_str()); return res; } @@ -15,9 +15,9 @@ TEST("make fixture macros") { EXPECT_FALSE(runPrint("../../apps/make_fixture_macros/vespalib_make_fixture_macros_app")); EXPECT_TRUE(runPrint("../../apps/make_fixture_macros/vespalib_make_fixture_macros_app 9 > macros.tmp")); - std::string diffCmd("diff -u "); + vespalib::string diffCmd("diff -u "); diffCmd += TEST_PATH("../../vespa/vespalib/testkit/generated_fixture_macros.h macros.tmp"); EXPECT_TRUE(runPrint(diffCmd.c_str())); } -TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/process/process_test.cpp b/vespalib/src/tests/process/process_test.cpp index 5f22e74ae4d..ec91bad4369 100644 --- a/vespalib/src/tests/process/process_test.cpp +++ b/vespalib/src/tests/process/process_test.cpp @@ -66,6 +66,46 @@ TEST(ProcessTest, proc_kill) { //----------------------------------------------------------------------------- +vespalib::string line1 = "this is a line"; +vespalib::string line2 = "this is also a line"; +vespalib::string line3 = "this is last line"; + +TEST(ProcessTest, read_line) { + Process proc("cat"); + for (const vespalib::string &line: {std::cref(line1), std::cref(line2), std::cref(line3)}) { + auto mem = proc.reserve(line.size() + 1); + memcpy(mem.data, line.data(), line.size()); + mem.data[line.size()] = '\n'; + proc.commit(line.size() + 1); + fprintf(stderr, "write: %s\n", line.c_str()); + auto res = proc.read_line(); + fprintf(stderr, "read: %s\n", line.c_str()); + EXPECT_EQ(res, line); + EXPECT_FALSE(proc.eof()); + } + proc.close(); + EXPECT_EQ(proc.read_line(), ""); + EXPECT_TRUE(proc.eof()); + EXPECT_EQ(proc.join(), 0); +} + +TEST(ProcessTest, read_line_without_newline) { + Process proc("cat"); + const auto &line = line3; + auto mem = proc.reserve(line.size()); + memcpy(mem.data, line.data(), line.size()); + proc.commit(line.size()); + fprintf(stderr, "write: %s\n", line.c_str()); + proc.close(); // need eof to flush line + auto res = proc.read_line(); + fprintf(stderr, "read: %s\n", line.c_str()); + EXPECT_EQ(res, line); + EXPECT_TRUE(proc.eof()); + EXPECT_EQ(proc.join(), 0); +} + +//----------------------------------------------------------------------------- + void write_slime(const Slime &slime, Output &out) { JsonFormat::encode(slime, out, true); out.reserve(1).data[0] = '\n'; diff --git a/vespalib/src/tests/tutorial/make_tutorial.cpp b/vespalib/src/tests/tutorial/make_tutorial.cpp index 46292a81005..58e3f529405 100644 --- a/vespalib/src/tests/tutorial/make_tutorial.cpp +++ b/vespalib/src/tests/tutorial/make_tutorial.cpp @@ -1,6 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/child_process.h> +#include <vespa/vespalib/process/process.h> #include <vespa/vespalib/util/stringfmt.h> #include <sys/mman.h> #include <sys/stat.h> @@ -10,48 +10,48 @@ using namespace vespalib; -std::string readFile(const std::string &filename) { +vespalib::string readFile(const vespalib::string &filename) { TEST_STATE(filename.c_str()); MappedFileInput file(filename); ASSERT_TRUE(file.valid()); Memory data = file.get(); - return std::string(data.data, data.size); + return vespalib::string(data.data, data.size); } -std::string runCommand(const std::string &cmd) { - std::string out; - ASSERT_TRUE(ChildProcess::run(cmd.c_str(), out)); +vespalib::string runCommand(const vespalib::string &cmd) { + vespalib::string out; + ASSERT_TRUE(Process::run(cmd, out)); return out; } -void insertExample(const std::string &name, const std::string &src_dir) { - std::string str = runCommand(make_string("%s/make_example.sh %s", src_dir.c_str(), +void insertExample(const vespalib::string &name, const vespalib::string &src_dir) { + vespalib::string str = runCommand(make_string("%s/make_example.sh %s", src_dir.c_str(), name.c_str())); fprintf(stdout, "%s", str.c_str()); } -void insertSource(const std::string &name, const std::string &src_dir) { - std::string str = runCommand(make_string("%s/make_source.sh %s", src_dir.c_str(), +void insertSource(const vespalib::string &name, const vespalib::string &src_dir) { + vespalib::string str = runCommand(make_string("%s/make_source.sh %s", src_dir.c_str(), name.c_str())); fprintf(stdout, "%s", str.c_str()); } -void insertFile(const std::string &name, const std::string &src_dir) { - std::string str = readFile(src_dir + "/" + name); +void insertFile(const vespalib::string &name, const vespalib::string &src_dir) { + vespalib::string str = readFile(src_dir + "/" + name); fprintf(stdout, "%s", str.c_str()); } -TEST_MAIN_WITH_PROCESS_PROXY() { - std::string pre("[insert:"); - std::string example("example:"); - std::string source("source:"); - std::string file("file:"); - std::string post("]\n"); +TEST_MAIN() { + vespalib::string pre("[insert:"); + vespalib::string example("example:"); + vespalib::string source("source:"); + vespalib::string file("file:"); + vespalib::string post("]\n"); size_t pos = 0; size_t end = 0; size_t cursor = 0; - std::string input = readFile(TEST_PATH("tutorial_source.html")); + vespalib::string input = readFile(TEST_PATH("tutorial_source.html")); while ((pos = input.find(pre, cursor)) < input.size() && (end = input.find(post, pos)) < input.size()) { @@ -59,15 +59,15 @@ TEST_MAIN_WITH_PROCESS_PROXY() { pos += pre.size(); if (input.find(example, pos) == pos) { pos += example.size(); - insertExample(std::string((input.data() + pos), (end - pos)), TEST_PATH("")); + insertExample(vespalib::string((input.data() + pos), (end - pos)), TEST_PATH("")); } else if (input.find(source, pos) == pos) { pos += source.size(); - insertSource(std::string((input.data() + pos), (end - pos)), TEST_PATH("")); + insertSource(vespalib::string((input.data() + pos), (end - pos)), TEST_PATH("")); } else if (input.find(file, pos) == pos) { pos += file.size(); - insertFile(std::string((input.data() + pos), (end - pos)), TEST_PATH("")); + insertFile(vespalib::string((input.data() + pos), (end - pos)), TEST_PATH("")); } else { - std::string str((input.data() + pos), (end - pos)); + vespalib::string str((input.data() + pos), (end - pos)); TEST_FATAL(make_string("invalid directive >%s<", str.c_str()).c_str()); } cursor = end + post.size(); diff --git a/vespalib/src/vespa/vespalib/process/process.cpp b/vespalib/src/vespa/vespalib/process/process.cpp index 3b202a830f5..7749ffdff65 100644 --- a/vespalib/src/vespa/vespalib/process/process.cpp +++ b/vespalib/src/vespa/vespalib/process/process.cpp @@ -104,6 +104,23 @@ Process::commit(size_t bytes) return *this; } +vespalib::string +Process::read_line() { + vespalib::string line; + for (auto mem = obtain(); (mem.size > 0); mem = obtain()) { + for (size_t i = 0; i < mem.size; ++i) { + if (mem.data[i] == '\n') { + evict(i + 1); + return line; + } else { + line.push_back(mem.data[i]); + } + } + evict(mem.size); + } + return line; +} + int Process::join() { diff --git a/vespalib/src/vespa/vespalib/process/process.h b/vespalib/src/vespa/vespalib/process/process.h index 97771752faa..d15784f5d31 100644 --- a/vespalib/src/vespa/vespalib/process/process.h +++ b/vespalib/src/vespa/vespalib/process/process.h @@ -43,6 +43,8 @@ public: Input &evict(size_t bytes) override; // Input (stdout) WritableMemory reserve(size_t bytes) override; // Output (stdin) Output &commit(size_t bytes) override; // Output (stdin) + vespalib::string read_line(); + bool eof() const { return _eof; } int join(); ~Process(); diff --git a/vespalib/src/vespa/vespalib/testkit/test_macros.h b/vespalib/src/vespa/vespalib/testkit/test_macros.h index 5ee8e170ef0..ebf3e06d283 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_macros.h +++ b/vespalib/src/vespa/vespalib/testkit/test_macros.h @@ -18,28 +18,24 @@ #define TEST_TRACE() TEST_MASTER.trace(__FILE__, __LINE__) #define TEST_THREAD(name) TEST_MASTER.setThreadName(name) #define TEST_BARRIER() TEST_MASTER.awaitThreadBarrier(__FILE__, __LINE__) -#define TEST_MAIN_IMPL(useProxy) \ - void test_kit_main(); \ - struct TestKitApp : FastOS_Application \ - { \ - bool useProcessStarter() const override { return useProxy; } \ - int Main() override; \ - }; \ - int main(int argc, char **argv) \ - { \ - TestKitApp app; \ - return app.Entry(argc, argv); \ - } \ - int TestKitApp::Main() { \ - TEST_MASTER.init(__FILE__); \ - test_kit_main(); \ - return (TEST_MASTER.fini() ? 0 : 1); \ - } \ +#define TEST_MAIN() \ + void test_kit_main(); \ + struct TestKitApp : FastOS_Application \ + { \ + int Main() override; \ + }; \ + int main(int argc, char **argv) \ + { \ + TestKitApp app; \ + return app.Entry(argc, argv); \ + } \ + int TestKitApp::Main() { \ + TEST_MASTER.init(__FILE__); \ + test_kit_main(); \ + return (TEST_MASTER.fini() ? 0 : 1); \ + } \ void test_kit_main() -#define TEST_MAIN() TEST_MAIN_IMPL(false) -#define TEST_MAIN_WITH_PROCESS_PROXY() TEST_MAIN_IMPL(true) - //----------------------------------------------------------------------------- #include "generated_fixture_macros.h" //----------------------------------------------------------------------------- diff --git a/vespalib/src/vespa/vespalib/testkit/testapp.h b/vespalib/src/vespa/vespalib/testkit/testapp.h index facb0857e8c..8ffb9958555 100644 --- a/vespalib/src/vespa/vespalib/testkit/testapp.h +++ b/vespalib/src/vespa/vespalib/testkit/testapp.h @@ -22,14 +22,6 @@ public: int Main() override; \ }; \ TEST_APPHOOK(test) -#define TEST_SETUP_WITHPROCESSPROXY(test) \ - class test : public vespalib::TestApp \ - { \ - public: \ - int Main(); \ - virtual bool useProcessStarter() const { return true; } \ - }; \ - TEST_APPHOOK(test) namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index 747ed736aad..f7bdd4427e3 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -14,7 +14,6 @@ vespa_add_library(vespalib_vespalib_util OBJECT binary_hamming_distance.cpp blockingthreadstackexecutor.cpp box.cpp - child_process.cpp classname.cpp compress.cpp compressor.cpp diff --git a/vespalib/src/vespa/vespalib/util/child_process.cpp b/vespalib/src/vespa/vespalib/util/child_process.cpp deleted file mode 100644 index 93db56cdf67..00000000000 --- a/vespalib/src/vespa/vespalib/util/child_process.cpp +++ /dev/null @@ -1,346 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "guard.h" -#include "child_process.h" -#include <cstring> -#include <vespa/vespalib/util/size_literals.h> - -namespace vespalib { - -namespace child_process { - -using namespace std::chrono; - -/** - * @brief ChildProcess internal timeout management. - **/ -class Timer -{ -private: - const steady_clock::time_point _startTime; - const int64_t _maxTimeMS; - milliseconds _elapsed; - -public: - Timer(int64_t maxTimeMS) - : _startTime(steady_clock::now()), - _maxTimeMS(maxTimeMS), - _elapsed(0) - { } - Timer &update() { - _elapsed = duration_cast<milliseconds>(steady_clock::now() - _startTime); - return *this; - } - int64_t elapsed() const { - return _elapsed.count(); - } - int64_t remaining() const { - if (_maxTimeMS == -1) { - return -1; - } - if (elapsed() > _maxTimeMS) { - return 0; - } - return (_maxTimeMS - _elapsed.count()); - } - int64_t waitTime() const { - int res = remaining(); - if (res >= 0 && res <= 10000) { - return res; - } - return 10000; - } - bool timeOut() const { - return (remaining() == 0); - } -}; - -} // namespace child_process - - -void -ChildProcess::Reader::OnReceiveData(const void *data, size_t length) -{ - const char *buf = (const char *) data; - std::unique_lock lock(_lock); - if (_gotEOF || (buf != nullptr && length == 0)) { // ignore special cases - return; - } - if (buf == nullptr) { // EOF - if (--_num_streams == 0) { - _gotEOF = true; - } - } else { - _queue.push(std::string(buf, length)); - } - if (_waitCnt > 0) { - _cond.notify_one(); - } -} - - -bool -ChildProcess::Reader::hasData() -{ - // NB: caller has lock on _cond - return (!_data.empty() || !_queue.empty()); -} - - -bool -ChildProcess::Reader::waitForData(child_process::Timer &timer, std::unique_lock<std::mutex> &guard) -{ - // NB: caller has lock on _cond - CounterGuard count(_waitCnt); - while (!timer.update().timeOut() && !hasData() && !_gotEOF) { - _cond.wait_for(guard, std::chrono::milliseconds(timer.waitTime())); - } - return hasData(); -} - - -void -ChildProcess::Reader::updateEOF() -{ - // NB: caller has lock on _cond - if (_data.empty() && _queue.empty() && _gotEOF) { - _readEOF = true; - } -} - - -ChildProcess::Reader::Reader(int num_streams) - : _lock(), - _cond(), - _queue(), - _data(), - _num_streams(num_streams), - _gotEOF(false), - _waitCnt(0), - _readEOF(false) -{ -} - - -ChildProcess::Reader::~Reader() = default; - - -uint32_t -ChildProcess::Reader::read(char *buf, uint32_t len, int msTimeout) -{ - if (eof()) { - return 0; - } - child_process::Timer timer(msTimeout); - std::unique_lock guard(_lock); - waitForData(timer, guard); - uint32_t bytes = 0; - while (bytes < len && hasData()) { - if (_data.empty()) { - _data = _queue.front(); - _queue.pop(); - } - if (len - bytes < _data.length()) { - memcpy(buf + bytes, _data.data(), len - bytes); - _data.erase(0, len - bytes); - bytes = len; - } else { - memcpy(buf + bytes, _data.data(), _data.length()); - bytes += _data.length(); - _data.clear(); - } - } - updateEOF(); - return bytes; -} - - -bool -ChildProcess::Reader::readLine(std::string &line, int msTimeout) -{ - line.clear(); - if (eof()) { - return false; - } - child_process::Timer timer(msTimeout); - std::unique_lock guard(_lock); - while (waitForData(timer, guard)) { - while (hasData()) { - if (_data.empty()) { - _data = _queue.front(); - _queue.pop(); - } - std::string::size_type ofs = _data.find('\n'); - if (ofs == std::string::npos) { - line.append(_data); - _data.clear(); - } else { - line.append(_data, 0, ofs); - _data.erase(0, ofs + 1); - updateEOF(); - return true; - } - } - } - updateEOF(); - if (eof()) { - return !line.empty(); - } - _data.swap(line); - return false; -} - -//----------------------------------------------------------------------------- - -void -ChildProcess::checkProc() -{ - if (_running) { - bool stillRunning; - if (_proc.PollWait(&_exitCode, &stillRunning) && !stillRunning) { - _running = false; - _failed = (_exitCode != 0); - } - } -} - - -ChildProcess::ChildProcess(const char *cmd) - : _reader(1), - _proc(cmd, &_reader), - _running(false), - _failed(false), - _exitCode(-918273645) -{ - _running = _proc.CreateWithShell(); - _failed = !_running; -} - -ChildProcess::ChildProcess(const char *cmd, capture_stderr_tag) - : _reader(2), - _proc(cmd, &_reader, &_reader), - _running(false), - _failed(false), - _exitCode(-918273645) -{ - _running = _proc.CreateWithShell(); - _failed = !_running; -} - - -ChildProcess::~ChildProcess() = default; - - -bool -ChildProcess::write(const char *buf, uint32_t len) -{ - if (len == 0) { - return true; - } - return _proc.WriteStdin(buf, len); -} - - -bool -ChildProcess::close() -{ - return _proc.WriteStdin(nullptr, 0); -} - - -uint32_t -ChildProcess::read(char *buf, uint32_t len, int msTimeout) -{ - return _reader.read(buf, len, msTimeout); -} - - -bool -ChildProcess::readLine(std::string &line, int msTimeout) -{ - return _reader.readLine(line, msTimeout); -} - - -bool -ChildProcess::wait(int msTimeout) -{ - bool done = true; - checkProc(); - if (_running) { - if (msTimeout != -1) { - msTimeout = (msTimeout + 999) / 1000; - } - if (_proc.Wait(&_exitCode, msTimeout)) { - _failed = (_exitCode != 0); - } else { - _failed = true; - done = false; - } - _running = false; - } - return done; -} - - -bool -ChildProcess::running() -{ - checkProc(); - return _running; -} - - -bool -ChildProcess::failed() -{ - checkProc(); - return _failed; -} - -int -ChildProcess::getExitCode() -{ - return _exitCode; -} - - -bool -ChildProcess::run(const std::string &input, const char *cmd, - std::string &output, int msTimeout) -{ - ChildProcess proc(cmd); - child_process::Timer timer(msTimeout); - char buf[4_Ki]; - proc.write(input.data(), input.length()); - proc.close(); // close stdin - while (!proc.eof() && !timer.timeOut()) { - uint32_t res = proc.read(buf, sizeof(buf), timer.remaining()); - output.append(buf, res); - timer.update(); - } - if ( ! output.empty() && output.find('\n') == output.size() - 1) { - output.erase(output.size() - 1, 1); - } - proc.wait(timer.update().remaining()); - return (!proc.running() && !proc.failed()); -} - - -bool -ChildProcess::run(const char *cmd, std::string &output, int msTimeout) -{ - std::string input; // empty input - return run(input, cmd, output, msTimeout); -} - - -bool -ChildProcess::run(const char *cmd, int msTimeout) -{ - std::string input; // empty input - std::string output; // ignore output - return run(input, cmd, output, msTimeout); -} - -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/util/child_process.h b/vespalib/src/vespa/vespalib/util/child_process.h deleted file mode 100644 index 877c56a8cb1..00000000000 --- a/vespalib/src/vespa/vespalib/util/child_process.h +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/fastos/process.h> -#ifndef FASTOS_NO_THREADS -#include <string> -#include <queue> -#include <condition_variable> - -namespace vespalib::child_process { class Timer; } - -namespace vespalib { -/** - * @brief Child Process utility class for running external programs - * - * Designed for use in unit tests and other places - * where you need to run, control and communicate with - * some external program. - **/ -class ChildProcess -{ -private: - class Reader : public FastOS_ProcessRedirectListener - { - private: - std::mutex _lock; - std::condition_variable _cond; - std::queue<std::string> _queue; - std::string _data; - int _num_streams; - bool _gotEOF; - int _waitCnt; - bool _readEOF; - - void OnReceiveData(const void *data, size_t length) override; - bool hasData(); - bool waitForData(child_process::Timer &timer, std::unique_lock<std::mutex> &lock); - void updateEOF(); - - public: - Reader(int num_streams); - ~Reader() override; - - uint32_t read(char *buf, uint32_t len, int msTimeout); - bool readLine(std::string &line, int msTimeout); - bool eof() const { return _readEOF; }; - }; - - Reader _reader; - FastOS_Process _proc; - bool _running; - bool _failed; - int _exitCode; - - void checkProc(); - -public: - ChildProcess(const ChildProcess &) = delete; - ChildProcess &operator=(const ChildProcess &) = delete; - struct capture_stderr_tag{}; - - /** - * @brief Run a child process - * - * Starts a process running the given command - * @param cmd A shell command line to run - **/ - explicit ChildProcess(const char *cmd); - - /** - * @brief Run a child process - * - * Starts a process running the given command. stderr is - * redirected into stdout. - * @param cmd A shell command line to run - **/ - explicit ChildProcess(const char *cmd, capture_stderr_tag); - - /** @brief destructor doing cleanup if needed */ - ~ChildProcess(); - - /** - * @return process id - **/ - pid_t getPid() { return _proc.GetProcessId(); } - - /** - * @brief send data as input to the running process - * - * The given data will be sent so it becomes - * available on the running process's standard input. - * - * @param buf the data containing len bytes to be sent - * @param len the number of bytes to send - * @return true if successful - **/ - bool write(const char *buf, uint32_t len); - - /** - * @brief close the running process's standard input - * - * when running a program that consumes stdin, - * make sure to call this method to signal - * that it can finish. - * @return true if successful - **/ - bool close(); - - /** - * @brief read program output - * - * If the running program writes data to its standard output, you - * can and should get the data with read() or readLine() calls. - * - * @param buf pointer where data is stored - * @param len number of bytes to try to read - * @param msTimeout number of milliseconds to wait for data - **/ - uint32_t read(char *buf, uint32_t len, int msTimeout = 10000); - - /** - * @brief read a line of program output - * - * See read(). - * @param line reference to a string where a line of program output will be stored - * @param msTimeout number of milliseconds to wait for data - * @return true if successful - **/ - bool readLine(std::string &line, int msTimeout = 10000); - - /** - * @brief check if the program has finished writing output - * @return true if standard output from the process is closed - **/ - bool eof() const { return _reader.eof(); } - - /** - * @brief wait for the program to exit - * @param msTimeout milliseconds to wait; the default (-1) will wait forever - * @return true if the program exited, false on timeout - **/ - bool wait(int msTimeout = -1); - - /** @brief check if the program is still running */ - bool running(); - - /** - * @brief get the exit code of the started program. - * - * Will return the stored exitcode. - * It assumes that the command is done. - * @return exit code. - **/ - int getExitCode(); - - /** - * @brief check if program failed - * - * Check if we failed to run the program or the program finished with - * a non-zero exit status. - * @return true if something went wrong - **/ - bool failed(); - - /** - * @brief run a command - * - * Utility function that runs the given command, sends input, and - * loops reading output until the program finishes or the timeout - * expires. - * Any final terminating newline will be erased from output, - * just like for shell backticks. - * - * @param input The input the program will receive - * @param cmd The command to run - * @param output Any output will be appended to this string - * @param msTimeout milliseconds timeout; -1 means wait forever for program to finish - **/ - static bool run(const std::string &input, const char *cmd, - std::string &output, int msTimeout = -1); - /** - * @brief run a command - * - * Utility function that runs the given command with no input, and - * loops reading output until the program finishes or the timeout - * expires. - * Any final terminating newline will be erased from output, - * just like for shell backticks. - * - * @param cmd The command to run - * @param output Any output will be appended to this string - * @param msTimeout milliseconds timeout; -1 means wait forever for program to finish - **/ - static bool run(const char *cmd, std::string &output, int msTimeout = -1); - - /** - * @brief run a command - * - * Utility function that runs the given command with no input, and - * loops reading output until the program finishes or the timeout - * expires. Output (if any) is ignored. - * - * @param cmd The command to run - * @param msTimeout milliseconds timeout; -1 means wait forever for program to finish - **/ - static bool run(const char *cmd, int msTimeout = -1); -}; - -} // namespace vespalib - -#endif // FASTOS_NO_THREADS diff --git a/vespalib/src/vespa/vespalib/util/signalhandler.cpp b/vespalib/src/vespa/vespalib/util/signalhandler.cpp index cd3e900ffe7..5f361ee3b01 100644 --- a/vespalib/src/vespa/vespalib/util/signalhandler.cpp +++ b/vespalib/src/vespa/vespalib/util/signalhandler.cpp @@ -59,7 +59,7 @@ SignalHandler::handleSignal(int signal) void SignalHandler::gotSignal() { - _gotSignal = 1; + _gotSignal.store(1, std::memory_order_relaxed); } SignalHandler::SignalHandler(int signal) @@ -97,13 +97,13 @@ SignalHandler::ignore() bool SignalHandler::check() const { - return (_gotSignal != 0); + return (_gotSignal.load(std::memory_order_relaxed) != 0); } void SignalHandler::clear() { - _gotSignal = 0; + _gotSignal.store(0, std::memory_order_relaxed); } void diff --git a/vespalib/src/vespa/vespalib/util/signalhandler.h b/vespalib/src/vespa/vespalib/util/signalhandler.h index abd9a8bc572..70abef00058 100644 --- a/vespalib/src/vespa/vespalib/util/signalhandler.h +++ b/vespalib/src/vespa/vespalib/util/signalhandler.h @@ -3,6 +3,7 @@ #include <csignal> #include <vector> +#include <atomic> namespace vespalib { @@ -35,7 +36,7 @@ private: * State indicating if the signal handled by this signal handler * has been caught. **/ - volatile sig_atomic_t _gotSignal; + std::atomic<int> _gotSignal; /** * Common signal handler for all caught signals. This method will diff --git a/vespamalloc/src/tests/doubledelete/expectsignal.cpp b/vespamalloc/src/tests/doubledelete/expectsignal.cpp index e9b71649268..66688a71923 100644 --- a/vespamalloc/src/tests/doubledelete/expectsignal.cpp +++ b/vespamalloc/src/tests/doubledelete/expectsignal.cpp @@ -1,6 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/child_process.h> +#include <vespa/vespalib/process/process.h> #include <sys/wait.h> using namespace vespalib; @@ -9,9 +9,6 @@ class Test : public TestApp { public: int Main() override; -private: - bool useProcessStarter() const override { return true; } - }; int Test::Main() @@ -25,14 +22,11 @@ int Test::Main() fprintf(stderr, "argc=%d : Running '%s' expecting signal %d\n", _argc, _argv[2], retval); - ChildProcess cmd(_argv[2]); - for(std::string line; cmd.readLine(line, 60000);) { + Process cmd(_argv[2]); + for (vespalib::string line = cmd.read_line(); !(line.empty() && cmd.eof()); line = cmd.read_line()) { fprintf(stdout, "%s\n", line.c_str()); } - - ASSERT_TRUE(cmd.wait(60000)); - - int exitCode = cmd.getExitCode(); + int exitCode = cmd.join(); if (exitCode == 65535) { fprintf(stderr, "[ERROR] child killed (timeout)\n"); diff --git a/vespamalloc/src/tests/overwrite/expectsignal.cpp b/vespamalloc/src/tests/overwrite/expectsignal.cpp index 904cedd479f..66688a71923 100644 --- a/vespamalloc/src/tests/overwrite/expectsignal.cpp +++ b/vespamalloc/src/tests/overwrite/expectsignal.cpp @@ -1,6 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/child_process.h> +#include <vespa/vespalib/process/process.h> #include <sys/wait.h> using namespace vespalib; @@ -9,8 +9,6 @@ class Test : public TestApp { public: int Main() override; -private: - virtual bool useProcessStarter() const override { return true; } }; int Test::Main() @@ -24,14 +22,11 @@ int Test::Main() fprintf(stderr, "argc=%d : Running '%s' expecting signal %d\n", _argc, _argv[2], retval); - ChildProcess cmd(_argv[2]); - for(std::string line; cmd.readLine(line, 60000);) { + Process cmd(_argv[2]); + for (vespalib::string line = cmd.read_line(); !(line.empty() && cmd.eof()); line = cmd.read_line()) { fprintf(stdout, "%s\n", line.c_str()); } - - ASSERT_TRUE(cmd.wait(60000)); - - int exitCode = cmd.getExitCode(); + int exitCode = cmd.join(); if (exitCode == 65535) { fprintf(stderr, "[ERROR] child killed (timeout)\n"); diff --git a/vespamalloc/src/tests/thread/thread.cpp b/vespamalloc/src/tests/thread/thread.cpp index b101b862268..d6f1ad5c9c0 100644 --- a/vespamalloc/src/tests/thread/thread.cpp +++ b/vespamalloc/src/tests/thread/thread.cpp @@ -11,8 +11,6 @@ class Test : public TestApp public: ~Test(); int Main() override; -private: - bool useIPCHelper() const override { return true; } }; Test::~Test() |