diff options
335 files changed, 3979 insertions, 2780 deletions
diff --git a/.copr/Makefile b/.copr/Makefile index b0322bf29b3..d515053b8bc 100644 --- a/.copr/Makefile +++ b/.copr/Makefile @@ -6,13 +6,17 @@ SOURCEDIR := $(RPMTOPDIR)/SOURCES SPECDIR := $(RPMTOPDIR)/SPECS SPECFILE := $(SPECDIR)/vespa-$(VESPA_VERSION).spec -srpm: +deps: dnf install -y git rpmdevtools - $(TOP)/../dist.sh $$(git tag --points-at HEAD | grep -oP "\d+\.\d+\.\d+" | sort -V | tail -1) - spectool -g -C $(SOURCEDIR) $(SPECFILE) - rpmbuild -bs --define "_topdir $(RPMTOPDIR)" $(SPECFILE) + +srpm: VESPA_VERSION = $$(git tag --points-at HEAD | grep -oP "\d+\.\d+\.\d+" | sort -V | tail -1) +srpm: deps + $(TOP)/../dist.sh $(VESPA_VERSION) + spectool -g -C $(SOURCEDIR) $(SPECDIR)/vespa-$(VESPA_VERSION).spec + rpmbuild -bs --define "_topdir $(RPMTOPDIR)" $(SPECDIR)/vespa-$(VESPA_VERSION).spec cp -a $(RPMTOPDIR)/SRPMS/* $(outdir) + clean: -rm -rf $(RPMTOPDIR) -.PHONY: srpm clean +.PHONY: clean deps srpm diff --git a/CMakeLists.txt b/CMakeLists.txt index 7d9968b6329..c9980fb1928 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -95,6 +95,7 @@ add_subdirectory(jdisc_jetty) add_subdirectory(jrt_test) add_subdirectory(juniper) add_subdirectory(linguistics) +add_subdirectory(linguistics-components) add_subdirectory(logd) add_subdirectory(logserver) add_subdirectory(logforwarder) diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceCluster.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceCluster.java index f7398d0478f..b4a0c3bd578 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceCluster.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceCluster.java @@ -113,6 +113,14 @@ public class ServiceCluster { } public String nodeDescription(boolean plural) { + return entityDescription("node", plural); + } + + public String serviceDescription(boolean plural) { + return entityDescription("service", plural); + } + + private String entityDescription(String entity, boolean plural) { String pluralSuffix = plural ? "s" : ""; return isConfigServer() ? "config server" + pluralSuffix : isConfigServerHost() ? "config server host" + pluralSuffix : @@ -121,7 +129,7 @@ public class ServiceCluster { isProxy() ? (plural ? "proxies" : "proxy") : isProxyHost() ? "proxy host" + pluralSuffix : isTenantHost() ? "tenant host" + pluralSuffix : - "node" + pluralSuffix + " of {" + serviceType + "," + clusterId + "}"; + entity + pluralSuffix + " of {" + serviceType + "," + clusterId + "}"; } private boolean isHostedVespaApplicationWithId(ApplicationInstanceId id) { diff --git a/client/go/Makefile b/client/go/Makefile index 17748d765c8..a86feb456ef 100644 --- a/client/go/Makefile +++ b/client/go/Makefile @@ -2,7 +2,7 @@ # The version to release. Defaults to the current tag or revision. # Use env VERSION=X.Y.Z make ... to override -VERSION ?= $(shell git describe --tags 2> /dev/null | sed -E "s/^vespa-|-1$$//g") +VERSION ?= $(shell git describe --tags 2> /dev/null | sed "s/^v//") DEVEL_VERSION := $(shell echo "0.0.0-`git rev-parse --short HEAD`") ifeq ($(VERSION),) VERSION = $(DEVEL_VERSION) @@ -26,10 +26,11 @@ all: test checkfmt install # # Example: # -# $ git checkout vespa-X.Y.Z-1 -# $ make dist-github +# $ git checkout vX.Y.Z +# $ make dist-homebrew dist-homebrew: dist-version - brew bump-formula-pr --tag vespa-$(VERSION)-1 --version $(VERSION) vespa-cli +# TODO(mpolden): Remove --version=0 after next release + brew bump-formula-pr --tag v$(VERSION) --version=0 vespa-cli # Create a GitHub release draft for all platforms. Note that this only creates a # draft, which is not publicly visible until it's explicitly published. @@ -40,7 +41,7 @@ dist-homebrew: dist-version # # Example: # -# $ git checkout vespa-X.Y.Z-1 +# $ git checkout vX.Y.Z # $ make dist-github dist-github: dist gh release create v$(VERSION) --repo vespa-engine/vespa --notes-file $(CURDIR)/README.md --draft --title "Vespa CLI $(VERSION)" \ @@ -83,7 +84,7 @@ dist-sha256sums: dist-version: ifeq ($(VERSION),$(DEVEL_VERSION)) - $(error Invalid release version: $(VERSION). Try 'git checkout vespa-X.Y.Z-1' or 'env VERSION=X.Y.Z make ...') + $(error Invalid release version: $(VERSION). Try 'git checkout vX.Y.Z' or 'env VERSION=X.Y.Z make ...') endif # diff --git a/client/go/cmd/api_key.go b/client/go/cmd/api_key.go index a838f1a05c8..b3284daa993 100644 --- a/client/go/cmd/api_key.go +++ b/client/go/cmd/api_key.go @@ -77,6 +77,6 @@ func printPublicKey(apiKeyFile, tenant string) { log.Printf("\nThis is your public key:\n%s", color.Green(pemPublicKey)) log.Printf("Its fingerprint is:\n%s\n", color.Cyan(fingerprint)) log.Print("\nTo use this key in Vespa Cloud click 'Add custom key' at") - log.Printf(color.Cyan("%s/tenant/%s/keys").String(), defaultConsoleURL, tenant) + log.Printf(color.Cyan("%s/tenant/%s/keys").String(), getConsoleURL(), tenant) log.Print("and paste the entire public key including the BEGIN and END lines.") } diff --git a/client/go/cmd/api_key_test.go b/client/go/cmd/api_key_test.go index 2497568604f..1deb628c21e 100644 --- a/client/go/cmd/api_key_test.go +++ b/client/go/cmd/api_key_test.go @@ -4,20 +4,20 @@ package cmd import ( - "strings" + "path/filepath" "testing" "github.com/stretchr/testify/assert" ) func TestAPIKey(t *testing.T) { - homeDir := t.TempDir() - keyFile := homeDir + "/.vespa/t1.api-key.pem" + homeDir := filepath.Join(t.TempDir(), ".vespa") + keyFile := filepath.Join(homeDir, "t1.api-key.pem") out, _ := execute(command{args: []string{"api-key", "-a", "t1.a1.i1"}, homeDir: homeDir}, t, nil) - assert.True(t, strings.HasPrefix(out, "Success: API private key written to "+keyFile+"\n")) + assert.Contains(t, out, "Success: API private key written to "+keyFile+"\n") out, _ = execute(command{args: []string{"api-key", "-a", "t1.a1.i1"}, homeDir: homeDir}, t, nil) - assert.True(t, strings.HasPrefix(out, "Error: File "+keyFile+" already exists\nHint: Use -f to overwrite it\n")) - assert.True(t, strings.Contains(out, "This is your public key")) + assert.Contains(t, out, "Error: File "+keyFile+" already exists\nHint: Use -f to overwrite it\n") + assert.Contains(t, out, "This is your public key") } diff --git a/client/go/cmd/cert_test.go b/client/go/cmd/cert_test.go index d93def2fa70..cd5f88764b9 100644 --- a/client/go/cmd/cert_test.go +++ b/client/go/cmd/cert_test.go @@ -14,7 +14,7 @@ import ( ) func TestCert(t *testing.T) { - homeDir := t.TempDir() + homeDir := filepath.Join(t.TempDir(), ".vespa") pkgDir := mockApplicationPackage(t, false) out, _ := execute(command{args: []string{"cert", "-a", "t1.a1.i1", pkgDir}, homeDir: homeDir}, t, nil) @@ -23,8 +23,8 @@ func TestCert(t *testing.T) { appDir := filepath.Join(pkgDir, "src", "main", "application") pkgCertificate := filepath.Join(appDir, "security", "clients.pem") - certificate := filepath.Join(homeDir, ".vespa", app.String(), "data-plane-public-cert.pem") - privateKey := filepath.Join(homeDir, ".vespa", app.String(), "data-plane-private-key.pem") + certificate := filepath.Join(homeDir, app.String(), "data-plane-public-cert.pem") + privateKey := filepath.Join(homeDir, app.String(), "data-plane-private-key.pem") assert.Equal(t, fmt.Sprintf("Success: Certificate written to %s\nSuccess: Certificate written to %s\nSuccess: Private key written to %s\n", pkgCertificate, certificate, privateKey), out) @@ -33,7 +33,7 @@ func TestCert(t *testing.T) { } func TestCertCompressedPackage(t *testing.T) { - homeDir := t.TempDir() + homeDir := filepath.Join(t.TempDir(), ".vespa") pkgDir := mockApplicationPackage(t, true) zipFile := filepath.Join(pkgDir, "target", "application.zip") err := os.MkdirAll(filepath.Dir(zipFile), 0755) diff --git a/client/go/cmd/clone.go b/client/go/cmd/clone.go index 9503a81debf..508ad49438f 100644 --- a/client/go/cmd/clone.go +++ b/client/go/cmd/clone.go @@ -6,11 +6,10 @@ package cmd import ( "archive/zip" + "errors" "io" - "io/ioutil" "log" "net/http" - "net/url" "os" "path/filepath" "strings" @@ -20,22 +19,29 @@ import ( "github.com/vespa-engine/vespa/client/go/util" ) -// Set this to test without downloading this file from github -var existingSampleAppsZip string +const sampleAppsCacheTTL = time.Hour * 168 // 1 week + var listApps bool +var forceClone bool func init() { rootCmd.AddCommand(cloneCmd) cloneCmd.Flags().BoolVarP(&listApps, "list", "l", false, "List available sample applications") + cloneCmd.Flags().BoolVarP(&forceClone, "force", "f", false, "Ignore cache and force downloading the latest sample application from GitHub") } var cloneCmd = &cobra.Command{ - // TODO: "application" and "list" subcommands? Use: "clone sample-application-path target-directory", Short: "Create files and directory structure for a new Vespa application from a sample application", - Long: `Creates an application package file structure. + Long: `Create files and directory structure for a new Vespa application +from a sample application. + +Sample applications are downloaded from +https://github.com/vespa-engine/sample-apps. -The application package is copied from a sample application in https://github.com/vespa-engine/sample-apps`, +By default sample applications are cached in the user's cache directory. This +directory can be overriden by setting the VESPA_CLI_CACHE_DIR environment +variable.`, Example: "$ vespa clone vespa-cloud/album-recommendation my-app", DisableAutoGenTag: true, Run: func(cmd *cobra.Command, args []string) { @@ -60,12 +66,7 @@ The application package is copied from a sample application in https://github.co func cloneApplication(source string, name string) { zipFile := getSampleAppsZip() - if zipFile == nil { - return - } - if existingSampleAppsZip == "" { // Indicates we created a temp file now - defer os.Remove(zipFile.Name()) - } + defer zipFile.Close() zipReader, zipOpenError := zip.OpenReader(zipFile.Name()) if zipOpenError != nil { @@ -101,45 +102,67 @@ func cloneApplication(source string, name string) { } } +func openOutputFile() (*os.File, error) { + cacheDir, err := vespaCliCacheDir() + if err != nil { + return nil, err + } + cacheFile := filepath.Join(cacheDir, "sample-apps-master.zip") + return os.OpenFile(cacheFile, os.O_RDWR|os.O_CREATE, 0755) +} + +func useCache(cacheFile *os.File) (bool, error) { + if forceClone { + return false, nil + } + stat, err := cacheFile.Stat() + if errors.Is(err, os.ErrNotExist) { + return false, nil + } else if err != nil { + return false, err + } + expiry := stat.ModTime().Add(sampleAppsCacheTTL) + return stat.Size() > 0 && time.Now().Before(expiry), nil +} + func getSampleAppsZip() *os.File { - if existingSampleAppsZip != "" { - existing, openExistingError := os.Open(existingSampleAppsZip) - if openExistingError != nil { - printErr(openExistingError, "Could not open existing sample apps zip file '", color.Cyan(existingSampleAppsZip), "'") - } - return existing + f, err := openOutputFile() + if err != nil { + fatalErr(err, "Could not determine location of cache file") + return nil + } + useCache, err := useCache(f) + if err != nil { + fatalErr(err, "Could not determine cache status", "Try ignoring the cache with the -f flag") + return nil + } + if useCache { + log.Print(color.Yellow("Using cached sample apps ...")) + return f } - // TODO: Cache it? log.Print(color.Yellow("Downloading sample apps ...")) // TODO: Spawn thread to indicate progress - zipUrl, _ := url.Parse("https://github.com/vespa-engine/sample-apps/archive/refs/heads/master.zip") - request := &http.Request{ - URL: zipUrl, - Method: "GET", + request, err := http.NewRequest("GET", "https://github.com/vespa-engine/sample-apps/archive/refs/heads/master.zip", nil) + if err != nil { + fatalErr(err, "Invalid URL") + return nil } - response, reqErr := util.HttpDo(request, time.Minute*60, "GitHub") - if reqErr != nil { - printErr(reqErr, "Could not download sample apps from GitHub") + response, err := util.HttpDo(request, time.Minute*60, "GitHub") + if err != nil { + fatalErr(err, "Could not download sample apps from GitHub") return nil } defer response.Body.Close() if response.StatusCode != 200 { - printErr(nil, "Could not download sample apps from GitHub: ", response.StatusCode) + fatalErr(nil, "Could not download sample apps from GitHub: ", response.StatusCode) return nil } - destination, tempFileError := ioutil.TempFile("", "prefix") - if tempFileError != nil { - printErr(tempFileError, "Could not create a temporary file to hold sample apps") - } - // destination, _ := os.Create("./" + name + "/sample-apps.zip") - // defer destination.Close() - _, err := io.Copy(destination, response.Body) - if err != nil { - printErr(err, "Could not download sample apps from GitHub") + if _, err := io.Copy(f, response.Body); err != nil { + fatalErr(err, "Could not write sample apps to file: ", f.Name()) return nil } - return destination + return f } func copy(f *zip.File, destinationDir string, zipEntryPrefix string) error { diff --git a/client/go/cmd/clone_test.go b/client/go/cmd/clone_test.go index 5027c5bf972..6cf11dd4d40 100644 --- a/client/go/cmd/clone_test.go +++ b/client/go/cmd/clone_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/vespa-engine/vespa/client/go/util" @@ -18,10 +19,14 @@ func TestClone(t *testing.T) { } func assertCreated(sampleAppName string, app string, t *testing.T) { - existingSampleAppsZip = "testdata/sample-apps-master.zip" - standardOut := executeCommand(t, &mockHttpClient{}, []string{"clone", sampleAppName, app}, []string{}) + testFile := filepath.Join("testdata", "sample-apps-master.zip") + now := time.Now() + if err := os.Chtimes(testFile, now, now); err != nil { // Ensure test file is considered new enough by cache mechanism + t.Fatal(err) + } + out, _ := execute(command{cacheDir: filepath.Dir(testFile), args: []string{"clone", sampleAppName, app}}, t, nil) defer os.RemoveAll(app) - assert.Equal(t, "Created "+app+"\n", standardOut) + assert.Equal(t, "Using cached sample apps ...\nCreated "+app+"\n", out) assert.True(t, util.PathExists(filepath.Join(app, "README.md"))) assert.True(t, util.PathExists(filepath.Join(app, "src", "main", "application"))) assert.True(t, util.IsDirectory(filepath.Join(app, "src", "main", "application"))) diff --git a/client/go/cmd/command_tester.go b/client/go/cmd/command_tester.go index f455ffa9957..6929b59decb 100644 --- a/client/go/cmd/command_tester.go +++ b/client/go/cmd/command_tester.go @@ -22,6 +22,7 @@ import ( type command struct { homeDir string + cacheDir string args []string moreArgs []string } @@ -31,12 +32,16 @@ func execute(cmd command, t *testing.T, client *mockHttpClient) (string, string) util.ActiveHttpClient = client } - // Set config dir. Use a separate one per test if none is specified + // Set Vespa CLI directories. Use a separate one per test if none is specified if cmd.homeDir == "" { - cmd.homeDir = t.TempDir() + cmd.homeDir = filepath.Join(t.TempDir(), ".vespa") viper.Reset() } - os.Setenv("VESPA_CLI_HOME", filepath.Join(cmd.homeDir, ".vespa")) + if cmd.cacheDir == "" { + cmd.cacheDir = filepath.Join(t.TempDir(), ".cache", "vespa") + } + os.Setenv("VESPA_CLI_HOME", cmd.homeDir) + os.Setenv("VESPA_CLI_CACHE_DIR", cmd.cacheDir) // Reset flags to their default value - persistent flags in Cobra persists over tests rootCmd.Flags().VisitAll(func(f *pflag.Flag) { @@ -111,5 +116,3 @@ func (c *mockHttpClient) Do(request *http.Request, timeout time.Duration) (*http } func (c *mockHttpClient) UseCertificate(certificate tls.Certificate) {} - -func convergeServices(client *mockHttpClient) { client.NextResponse(200, `{"converged":true}`) } diff --git a/client/go/cmd/config.go b/client/go/cmd/config.go index 3753d9a9390..863f247bd7c 100644 --- a/client/go/cmd/config.go +++ b/client/go/cmd/config.go @@ -34,8 +34,16 @@ func init() { } var configCmd = &cobra.Command{ - Use: "config", - Short: "Configure default values for flags", + Use: "config", + Short: "Configure persistent values for flags", + Long: `Configure persistent values for flags. + +This command allows setting a persistent value for a given flag. On future +invocations the flag can then be omitted as it is read from the config file +instead. + +Configuration is written to $HOME/.vespa by default. This path can be +overridden by setting the VESPA_CLI_HOME environment variable.`, DisableAutoGenTag: true, Run: func(cmd *cobra.Command, args []string) { // Root command does nothing @@ -100,16 +108,8 @@ type Config struct { } func LoadConfig() (*Config, error) { - home := os.Getenv("VESPA_CLI_HOME") - if home == "" { - var err error - home, err = os.UserHomeDir() - if err != nil { - return nil, err - } - home = filepath.Join(home, ".vespa") - } - if err := os.MkdirAll(home, 0700); err != nil { + home, err := vespaCliHome() + if err != nil { return nil, err } c := &Config{Home: home, createDirs: true} diff --git a/client/go/cmd/config_test.go b/client/go/cmd/config_test.go index cf50f561f0f..25ba7cc0655 100644 --- a/client/go/cmd/config_test.go +++ b/client/go/cmd/config_test.go @@ -1,13 +1,14 @@ package cmd import ( + "path/filepath" "testing" "github.com/stretchr/testify/assert" ) func TestConfig(t *testing.T) { - homeDir := t.TempDir() + homeDir := filepath.Join(t.TempDir(), ".vespa") assertConfigCommand(t, "invalid option or value: \"foo\": \"bar\"\n", homeDir, "config", "set", "foo", "bar") assertConfigCommand(t, "foo = <unset>\n", homeDir, "config", "get", "foo") assertConfigCommand(t, "target = local\n", homeDir, "config", "get", "target") diff --git a/client/go/cmd/curl_test.go b/client/go/cmd/curl_test.go index 340eacd0bd3..d5021e19cf2 100644 --- a/client/go/cmd/curl_test.go +++ b/client/go/cmd/curl_test.go @@ -10,13 +10,12 @@ import ( ) func TestCurl(t *testing.T) { - homeDir := t.TempDir() + homeDir := filepath.Join(t.TempDir(), ".vespa") httpClient := &mockHttpClient{} - convergeServices(httpClient) out, _ := execute(command{homeDir: homeDir, args: []string{"curl", "-n", "-a", "t1.a1.i1", "--", "-v", "--data-urlencode", "arg=with space", "/search"}}, t, httpClient) expected := fmt.Sprintf("curl --key %s --cert %s -v --data-urlencode 'arg=with space' https://127.0.0.1:8080/search\n", - filepath.Join(homeDir, ".vespa", "t1.a1.i1", "data-plane-private-key.pem"), - filepath.Join(homeDir, ".vespa", "t1.a1.i1", "data-plane-public-cert.pem")) + filepath.Join(homeDir, "t1.a1.i1", "data-plane-private-key.pem"), + filepath.Join(homeDir, "t1.a1.i1", "data-plane-public-cert.pem")) assert.Equal(t, expected, out) } diff --git a/client/go/cmd/deploy.go b/client/go/cmd/deploy.go index 9bf59187778..b3171d184e0 100644 --- a/client/go/cmd/deploy.go +++ b/client/go/cmd/deploy.go @@ -39,7 +39,11 @@ When this returns successfully the application package has been validated and activated on config servers. The process of applying it on individual nodes has started but may not have completed. -If application directory is not specified, it defaults to working directory.`, +If application directory is not specified, it defaults to working directory. + +When deploying to Vespa Cloud the system can be overridden by setting the +environment variable VESPA_CLI_CLOUD_SYSTEM. This is intended for internal use +only.`, Example: "$ vespa deploy .", Args: cobra.MaximumNArgs(1), DisableAutoGenTag: true, @@ -78,7 +82,7 @@ If application directory is not specified, it defaults to working directory.`, if opts.IsCloud() { log.Printf("\nUse %s for deployment status, or follow this deployment at", color.Cyan("vespa status")) log.Print(color.Cyan(fmt.Sprintf("%s/tenant/%s/application/%s/dev/instance/%s/job/%s-%s/run/%d", - defaultConsoleURL, + getConsoleURL(), opts.Deployment.Application.Tenant, opts.Deployment.Application.Application, opts.Deployment.Application.Instance, opts.Deployment.Zone.Environment, opts.Deployment.Zone.Region, sessionOrRunID))) diff --git a/client/go/cmd/deploy_test.go b/client/go/cmd/deploy_test.go index 443f7e8846f..9614806b968 100644 --- a/client/go/cmd/deploy_test.go +++ b/client/go/cmd/deploy_test.go @@ -130,7 +130,7 @@ func assertActivate(applicationPackage string, arguments []string, t *testing.T) if err := cfg.WriteSessionID(vespa.DefaultApplication, 42); err != nil { t.Fatal(err) } - out, _ := execute(command{args: arguments, homeDir: homeDir}, t, client) + out, _ := execute(command{args: arguments, homeDir: cfg.Home}, t, client) assert.Equal(t, "Success: Activated "+applicationPackage+" with session 42\n", out) diff --git a/client/go/cmd/document_test.go b/client/go/cmd/document_test.go index 8aecb538f89..1f82b85f915 100644 --- a/client/go/cmd/document_test.go +++ b/client/go/cmd/document_test.go @@ -67,7 +67,6 @@ func TestDocumentRemoveWithoutIdArg(t *testing.T) { func TestDocumentSendMissingId(t *testing.T) { arguments := []string{"document", "put", "testdata/A-Head-Full-of-Dreams-Without-Operation.json"} client := &mockHttpClient{} - convergeServices(client) assert.Equal(t, "Error: No document id given neither as argument or as a 'put' key in the json file\n", executeCommand(t, client, arguments, []string{})) @@ -76,7 +75,6 @@ func TestDocumentSendMissingId(t *testing.T) { func TestDocumentSendWithDisagreeingOperations(t *testing.T) { arguments := []string{"document", "update", "testdata/A-Head-Full-of-Dreams-Put.json"} client := &mockHttpClient{} - convergeServices(client) assert.Equal(t, "Error: Wanted document operation is update but the JSON file specifies put\n", executeCommand(t, client, arguments, []string{})) @@ -140,7 +138,6 @@ func assertDocumentGet(arguments []string, documentId string, t *testing.T) { func assertDocumentError(t *testing.T, status int, errorMessage string) { client := &mockHttpClient{} - convergeServices(client) client.NextResponse(status, errorMessage) assert.Equal(t, "Error: Invalid document operation: Status "+strconv.Itoa(status)+"\n\n"+errorMessage+"\n", @@ -151,7 +148,6 @@ func assertDocumentError(t *testing.T, status int, errorMessage string) { func assertDocumentServerError(t *testing.T, status int, errorMessage string) { client := &mockHttpClient{} - convergeServices(client) client.NextResponse(status, errorMessage) assert.Equal(t, "Error: Container (document API) at 127.0.0.1:8080: Status "+strconv.Itoa(status)+"\n\n"+errorMessage+"\n", @@ -161,6 +157,5 @@ func assertDocumentServerError(t *testing.T, status int, errorMessage string) { } func documentServiceURL(client *mockHttpClient) string { - convergeServices(client) return getService("document", 0).BaseURL } diff --git a/client/go/cmd/helpers.go b/client/go/cmd/helpers.go index f29a842aed2..98d6814d16f 100644 --- a/client/go/cmd/helpers.go +++ b/client/go/cmd/helpers.go @@ -10,14 +10,13 @@ import ( "io/ioutil" "log" "os" + "path/filepath" "strings" "time" "github.com/vespa-engine/vespa/client/go/vespa" ) -const defaultConsoleURL = "https://console.vespa.oath.cloud" - var exitFunc = os.Exit // To allow overriding Exit in tests func fatalErrHint(err error, hints ...string) { @@ -50,6 +49,36 @@ func printSuccess(msg ...interface{}) { log.Print(color.Green("Success: "), fmt.Sprint(msg...)) } +func vespaCliHome() (string, error) { + home := os.Getenv("VESPA_CLI_HOME") + if home == "" { + userHome, err := os.UserHomeDir() + if err != nil { + return "", err + } + home = filepath.Join(userHome, ".vespa") + } + if err := os.MkdirAll(home, 0700); err != nil { + return "", err + } + return home, nil +} + +func vespaCliCacheDir() (string, error) { + cacheDir := os.Getenv("VESPA_CLI_CACHE_DIR") + if cacheDir == "" { + userCacheDir, err := os.UserCacheDir() + if err != nil { + return "", err + } + cacheDir = filepath.Join(userCacheDir, "vespa") + } + if err := os.MkdirAll(cacheDir, 0755); err != nil { + return "", err + } + return cacheDir, nil +} + func deploymentFromArgs() vespa.Deployment { zone, err := vespa.ZoneFromString(zoneArg) if err != nil { @@ -102,18 +131,32 @@ func getService(service string, sessionOrRunID int64) *vespa.Service { t := getTarget() timeout := time.Duration(waitSecsArg) * time.Second if timeout > 0 { - log.Printf("Waiting up to %d %s for services to become available ...", color.Cyan(waitSecsArg), color.Cyan("seconds")) - } - if err := t.DiscoverServices(timeout, sessionOrRunID); err != nil { - fatalErr(err, "Services unavailable") + log.Printf("Waiting up to %d %s for service to become available ...", color.Cyan(waitSecsArg), color.Cyan("seconds")) } - s, err := t.Service(service) + s, err := t.Service(service, timeout, sessionOrRunID) if err != nil { - fatalErr(err, "Invalid service") + fatalErr(err, "Invalid service: ", service) } return s } +func getConsoleURL() string { + system := os.Getenv("VESPA_CLI_CLOUD_SYSTEM") + if system == "publiccd" { + return "https://console-cd.vespa.oath.cloud" + } + return "https://console.vespa.oath.cloud" + +} + +func getApiURL() string { + system := os.Getenv("VESPA_CLI_CLOUD_SYSTEM") + if system == "publiccd" { + return "https://api.vespa-external-cd.aws.oath.cloud:4443" + } + return "https://api.vespa-external.aws.oath.cloud:4443" +} + func getTarget() vespa.Target { targetType := getTargetType() if strings.HasPrefix(targetType, "http") { @@ -147,7 +190,7 @@ func getTarget() vespa.Target { if err != nil { fatalErrHint(err, "Deployment to cloud requires a certificate. Try 'vespa cert'") } - return vespa.CloudTarget(deployment, apiKey, + return vespa.CloudTarget(getApiURL(), deployment, apiKey, vespa.TLSOptions{ KeyPair: kp, CertificateFile: certificateFile, diff --git a/client/go/cmd/query_test.go b/client/go/cmd/query_test.go index bd9ae91f24d..137ffa01cd5 100644 --- a/client/go/cmd/query_test.go +++ b/client/go/cmd/query_test.go @@ -56,7 +56,6 @@ func assertQuery(t *testing.T, expectedQuery string, query ...string) { func assertQueryError(t *testing.T, status int, errorMessage string) { client := &mockHttpClient{} - convergeServices(client) client.NextResponse(status, errorMessage) assert.Equal(t, "Error: Invalid query: Status "+strconv.Itoa(status)+"\n"+errorMessage+"\n", @@ -66,7 +65,6 @@ func assertQueryError(t *testing.T, status int, errorMessage string) { func assertQueryServiceError(t *testing.T, status int, errorMessage string) { client := &mockHttpClient{} - convergeServices(client) client.NextResponse(status, errorMessage) assert.Equal(t, "Error: Status "+strconv.Itoa(status)+" from container at 127.0.0.1:8080\n"+errorMessage+"\n", @@ -75,6 +73,5 @@ func assertQueryServiceError(t *testing.T, status int, errorMessage string) { } func queryServiceURL(client *mockHttpClient) string { - convergeServices(client) return getService("query", 0).BaseURL } diff --git a/client/go/cmd/status_test.go b/client/go/cmd/status_test.go index 8ddca71a35b..0c1c8e4e3a7 100644 --- a/client/go/cmd/status_test.go +++ b/client/go/cmd/status_test.go @@ -44,7 +44,6 @@ func TestStatusErrorResponse(t *testing.T) { func assertDeployStatus(target string, args []string, t *testing.T) { client := &mockHttpClient{} - convergeServices(client) assert.Equal(t, "Deploy API at "+target+" is ready\n", executeCommand(t, client, []string{"status", "deploy"}, args), @@ -54,14 +53,12 @@ func assertDeployStatus(target string, args []string, t *testing.T) { func assertQueryStatus(target string, args []string, t *testing.T) { client := &mockHttpClient{} - convergeServices(client) assert.Equal(t, "Container (query API) at "+target+" is ready\n", executeCommand(t, client, []string{"status", "query"}, args), "vespa status container") assert.Equal(t, target+"/ApplicationStatus", client.lastRequest.URL.String()) - convergeServices(client) assert.Equal(t, "Container (query API) at "+target+" is ready\n", executeCommand(t, client, []string{"status"}, args), @@ -71,7 +68,6 @@ func assertQueryStatus(target string, args []string, t *testing.T) { func assertDocumentStatus(target string, args []string, t *testing.T) { client := &mockHttpClient{} - convergeServices(client) assert.Equal(t, "Container (document API) at "+target+" is ready\n", executeCommand(t, client, []string{"status", "document"}, args), @@ -81,7 +77,6 @@ func assertDocumentStatus(target string, args []string, t *testing.T) { func assertQueryStatusError(target string, args []string, t *testing.T) { client := &mockHttpClient{} - convergeServices(client) client.NextStatus(500) assert.Equal(t, "Container (query API) at "+target+" is not ready\nStatus 500\n", diff --git a/client/go/vespa/deploy.go b/client/go/vespa/deploy.go index ece841617c0..19319724d18 100644 --- a/client/go/vespa/deploy.go +++ b/client/go/vespa/deploy.go @@ -71,7 +71,7 @@ func (d DeploymentOpts) String() string { func (d *DeploymentOpts) IsCloud() bool { return d.Target.Type() == cloudTargetType } func (d *DeploymentOpts) url(path string) (*url.URL, error) { - service, err := d.Target.Service("deploy") + service, err := d.Target.Service(deployService, 0, 0) if err != nil { return nil, err } diff --git a/client/go/vespa/target.go b/client/go/vespa/target.go index 69dc876c1c8..aa0ddb8babb 100644 --- a/client/go/vespa/target.go +++ b/client/go/vespa/target.go @@ -24,8 +24,6 @@ const ( queryService = "query" documentService = "document" - defaultCloudAPI = "https://api.vespa-external.aws.oath.cloud:4443" - waitRetryInterval = 2 * time.Second ) @@ -41,11 +39,8 @@ type Target interface { // Type returns this target's type, e.g. local or cloud. Type() string - // Service returns the service for given name. - Service(name string) (*Service, error) - - // DiscoverServices queries for services available on this target after the deployment run has completed. - DiscoverServices(timeout time.Duration, runID int64) error + // Service returns the service for given name. If timeout is non-zero, wait for the service to converge. + Service(name string, timeout time.Duration, sessionOrRunID int64) (*Service, error) } // TLSOptions configures the certificate to use for service requests. @@ -107,7 +102,12 @@ func (s *Service) Description() string { func (t *customTarget) Type() string { return t.targetType } -func (t *customTarget) Service(name string) (*Service, error) { +func (t *customTarget) Service(name string, timeout time.Duration, sessionID int64) (*Service, error) { + if timeout > 0 && name != deployService { + if err := t.waitForConvergence(timeout); err != nil { + return nil, err + } + } switch name { case deployService, queryService, documentService: url, err := t.urlWithPort(name) @@ -139,12 +139,12 @@ func (t *customTarget) urlWithPort(serviceName string) (string, error) { return u.String(), nil } -func (t *customTarget) DiscoverServices(timeout time.Duration, runID int64) error { - deployService, err := t.Service("deploy") +func (t *customTarget) waitForConvergence(timeout time.Duration) error { + deployer, err := t.Service(deployService, 0, 0) if err != nil { return err } - url := fmt.Sprintf("%s/application/v2/tenant/default/application/default/environment/prod/region/default/instance/default/serviceconverge", deployService.BaseURL) + url := fmt.Sprintf("%s/application/v2/tenant/default/application/default/environment/prod/region/default/instance/default/serviceconverge", deployer.BaseURL) req, err := http.NewRequest("GET", url, nil) if err != nil { return err @@ -171,7 +171,7 @@ func (t *customTarget) DiscoverServices(timeout time.Duration, runID int64) erro } type cloudTarget struct { - cloudAPI string + apiURL string targetType string deployment Deployment apiKey []byte @@ -184,27 +184,30 @@ type cloudTarget struct { func (t *cloudTarget) Type() string { return t.targetType } -func (t *cloudTarget) Service(name string) (*Service, error) { +func (t *cloudTarget) Service(name string, timeout time.Duration, runID int64) (*Service, error) { + if name != deployService { + if err := t.waitForEndpoints(timeout, runID); err != nil { + return nil, err + } + } switch name { case deployService: - return &Service{Name: name, BaseURL: t.cloudAPI}, nil + return &Service{Name: name, BaseURL: t.apiURL}, nil case queryService: if t.queryURL == "" { - return nil, fmt.Errorf("service %s not discovered", name) + return nil, fmt.Errorf("service %s is not discovered", name) } return &Service{Name: name, BaseURL: t.queryURL, TLSOptions: t.tlsOptions}, nil case documentService: if t.documentURL == "" { - return nil, fmt.Errorf("service %s not discovered", name) + return nil, fmt.Errorf("service %s is not discovered", name) } return &Service{Name: name, BaseURL: t.documentURL, TLSOptions: t.tlsOptions}, nil } return nil, fmt.Errorf("unknown service: %s", name) } -// DiscoverServices waits for run identified by runID to complete and at least one endpoint is available, or timeout -// passes. -func (t *cloudTarget) DiscoverServices(timeout time.Duration, runID int64) error { +func (t *cloudTarget) waitForEndpoints(timeout time.Duration, runID int64) error { signer := NewRequestSigner(t.deployment.Application.SerializedForm(), t.apiKey) if runID > 0 { if err := t.waitForRun(signer, runID, timeout); err != nil { @@ -216,7 +219,7 @@ func (t *cloudTarget) DiscoverServices(timeout time.Duration, runID int64) error func (t *cloudTarget) waitForRun(signer *RequestSigner, runID int64, timeout time.Duration) error { runURL := fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/job/%s-%s/run/%d", - t.cloudAPI, + t.apiURL, t.deployment.Application.Tenant, t.deployment.Application.Application, t.deployment.Application.Instance, t.deployment.Zone.Environment, t.deployment.Zone.Region, runID) req, err := http.NewRequest("GET", runURL, nil) @@ -234,8 +237,8 @@ func (t *cloudTarget) waitForRun(signer *RequestSigner, runID int64, timeout tim return req } jobSuccessFunc := func(status int, response []byte) (bool, error) { - if status/100 != 2 { - return false, nil + if ok, err := isOK(status); !ok { + return ok, err } var resp jobResponse if err := json.Unmarshal(response, &resp); err != nil { @@ -280,7 +283,7 @@ func (t *cloudTarget) printLog(response jobResponse, last int64) int64 { func (t *cloudTarget) discoverEndpoints(signer *RequestSigner, timeout time.Duration) error { deploymentURL := fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/environment/%s/region/%s", - t.cloudAPI, + t.apiURL, t.deployment.Application.Tenant, t.deployment.Application.Application, t.deployment.Application.Instance, t.deployment.Zone.Environment, t.deployment.Zone.Region) req, err := http.NewRequest("GET", deploymentURL, nil) @@ -292,8 +295,8 @@ func (t *cloudTarget) discoverEndpoints(signer *RequestSigner, timeout time.Dura } var endpointURL string endpointFunc := func(status int, response []byte) (bool, error) { - if status/100 != 2 { - return false, nil + if ok, err := isOK(status); !ok { + return ok, err } var resp deploymentResponse if err := json.Unmarshal(response, &resp); err != nil { @@ -316,6 +319,13 @@ func (t *cloudTarget) discoverEndpoints(signer *RequestSigner, timeout time.Dura return nil } +func isOK(status int) (bool, error) { + if status == 401 { + return false, fmt.Errorf("status %d: invalid api key", status) + } + return status/100 == 2, nil +} + // LocalTarget creates a target for a Vespa platform running locally. func LocalTarget() Target { return &customTarget{targetType: localTargetType, baseURL: "http://127.0.0.1"} @@ -327,9 +337,9 @@ func CustomTarget(baseURL string) Target { } // CloudTarget creates a Target for the Vespa Cloud platform. -func CloudTarget(deployment Deployment, apiKey []byte, tlsOptions TLSOptions, logOptions LogOptions) Target { +func CloudTarget(apiURL string, deployment Deployment, apiKey []byte, tlsOptions TLSOptions, logOptions LogOptions) Target { return &cloudTarget{ - cloudAPI: defaultCloudAPI, + apiURL: apiURL, targetType: cloudTargetType, deployment: deployment, apiKey: apiKey, @@ -409,7 +419,8 @@ func wait(fn responseFunc, reqFn requestFunc, certificate *tls.Certificate, time return statusCode, nil } } - if loopOnce { + timeLeft := deadline.Sub(time.Now()) + if loopOnce || timeLeft < waitRetryInterval { break } time.Sleep(waitRetryInterval) diff --git a/client/go/vespa/target_test.go b/client/go/vespa/target_test.go index 31f145f0db3..2c90baefbbc 100644 --- a/client/go/vespa/target_test.go +++ b/client/go/vespa/target_test.go @@ -74,11 +74,11 @@ func TestCustomTargetWait(t *testing.T) { defer srv.Close() target := CustomTarget(srv.URL) - err := target.DiscoverServices(0, 42) + _, err := target.Service("query", time.Millisecond, 42) assert.NotNil(t, err) vc.deploymentConverged = true - err = target.DiscoverServices(0, 42) + _, err = target.Service("query", time.Millisecond, 42) assert.Nil(t, err) assertServiceWait(t, 200, target, "deploy") @@ -102,6 +102,7 @@ func TestCloudTargetWait(t *testing.T) { var logWriter bytes.Buffer target := CloudTarget( + "https://example.com", Deployment{ Application: ApplicationID{Tenant: "t1", Application: "a1", Instance: "i1"}, Zone: ZoneID{Environment: "dev", Region: "us-north-1"}, @@ -110,20 +111,17 @@ func TestCloudTargetWait(t *testing.T) { TLSOptions{KeyPair: x509KeyPair}, LogOptions{Writer: &logWriter}) if ct, ok := target.(*cloudTarget); ok { - ct.cloudAPI = srv.URL + ct.apiURL = srv.URL } else { t.Fatalf("Wrong target type %T", ct) } assertServiceWait(t, 200, target, "deploy") - _, err = target.Service("query") - assert.NotNil(t, err) - - err = target.DiscoverServices(0, 42) + _, err = target.Service("query", time.Millisecond, 42) assert.NotNil(t, err) vc.deploymentConverged = true - err = target.DiscoverServices(0, 42) + _, err = target.Service("query", time.Millisecond, 42) assert.Nil(t, err) assertServiceWait(t, 500, target, "query") @@ -136,13 +134,13 @@ func TestCloudTargetWait(t *testing.T) { } func assertServiceURL(t *testing.T, url string, target Target, service string) { - s, err := target.Service(service) + s, err := target.Service(service, 0, 42) assert.Nil(t, err) assert.Equal(t, url, s.BaseURL) } func assertServiceWait(t *testing.T, expectedStatus int, target Target, service string) { - s, err := target.Service(service) + s, err := target.Service(service, 0, 42) assert.Nil(t, err) status, err := s.Wait(0) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 9ee36831d6a..6d5c7a27a47 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -80,13 +80,12 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useAsyncMessageHandlingOnSchedule() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double feedConcurrency() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default int metricsproxyNumThreads() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"baldersheim"}) default boolean enforceRankProfileInheritance() { return false; } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.472") default boolean enforceRankProfileInheritance() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}) default int largeRankExpressionLimit() { return 8192; } - @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.468") default boolean useExternalRankExpressions() { return true; } - @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.468") default boolean distributeExternalRankExpressions() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxConcurrentMergesPerNode() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxMergeQueueSize() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"baldersheim"}) default boolean dryRunOnnxOnSetup() { return true; } + @ModelFeatureFlag(owners = {"baldersheim"}) default boolean containerDumpHeapOnShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"baldersheim"}) default double containerShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"geirst"}) default boolean enableFeedBlockInDistributor() { return true; } @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default List<String> allowedAthenzProxyIdentities() { return List.of(); } @ModelFeatureFlag(owners = {"vekterli"}) default int maxActivationInhibitedOutOfSyncGroups() { return 0; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 4bf20e75a5d..e27e0e7624f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -63,6 +63,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double resourceLimitDisk = 0.8; private double resourceLimitMemory = 0.8; private double minNodeRatioPerGroup = 0.0; + private boolean containerDumpHeapOnShutdownTimeout = false; + private double containerShutdownTimeout = 50.0; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -105,10 +107,14 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public double resourceLimitMemory() { return resourceLimitMemory; } @Override public double minNodeRatioPerGroup() { return minNodeRatioPerGroup; } @Override public int metricsproxyNumThreads() { return 1; } - @Override public boolean enforceRankProfileInheritance() { return enforceRankProfileInheritance; } - - public TestProperties enforceRankProfileInheritance(boolean value) { - enforceRankProfileInheritance = value; + @Override public double containerShutdownTimeout() { return containerShutdownTimeout; } + @Override public boolean containerDumpHeapOnShutdownTimeout() { return containerDumpHeapOnShutdownTimeout; } + public TestProperties containerDumpHeapOnShutdownTimeout(boolean value) { + containerDumpHeapOnShutdownTimeout = value; + return this; + } + public TestProperties containerShutdownTimeout(double value) { + containerShutdownTimeout = value; return this; } public TestProperties largeRankExpressionLimit(int value) { diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java index 2784c111019..decc6e98bc4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java @@ -223,15 +223,11 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce didApply = parent.addUserConfig(builder); } - if (log.isLoggable(Level.FINEST)) { - log.log(Level.FINEST, "User configs is: " + userConfigs.toString()); - } + log.log(Level.FINEST, () -> "User configs is: " + userConfigs.toString()); // TODO: What do we do with md5. Currently ignored for user configs? ConfigDefinitionKey key = new ConfigDefinitionKey(builder.getDefName(), builder.getDefNamespace()); if (userConfigs.get(key) != null) { - if (log.isLoggable(Level.FINEST)) { - log.log(Level.FINEST, "Apply in " + configId); - } + log.log(Level.FINEST, () -> "Apply in " + configId); applyUserConfig(builder, userConfigs.get(key)); didApply = true; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java index 1719ea72cb0..472bc9d5413 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java @@ -11,6 +11,7 @@ import java.util.Collection; import java.util.Objects; public class DistributableResource { + public enum PathType { FILE, URI, BLOB }; /** The search definition-unique name of this constant */ @@ -95,10 +96,9 @@ public class DistributableResource { } } + @Override public String toString() { - StringBuilder b = new StringBuilder(); - b.append("resource '").append(name).append(" of type '").append(pathType) - .append("' with ref '").append(fileReference).append("'"); - return b.toString(); + return "resource '" + name + " of type '" + pathType + "' with ref '" + fileReference + "'"; } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 9d51d39f3d0..d169760538d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -227,12 +227,7 @@ public class RankProfile implements Cloneable { String msg = "rank-profile '" + getName() + "' inherits '" + inheritedName + "', but it does not exist anywhere in the inheritance of search '" + ((getSearch() != null) ? getSearch().getName() : " global rank profiles") + "'."; - if (search.getDeployProperties().featureFlags().enforceRankProfileInheritance()) { - throw new IllegalArgumentException(msg); - } else { - deployLogger.logApplicationPackage(Level.WARNING, msg); - inherited = resolveIndependentOfInheritance(); - } + throw new IllegalArgumentException(msg); } else { List<String> children = new ArrayList<>(); children.add(createFullyQualifiedName()); @@ -241,12 +236,7 @@ public class RankProfile implements Cloneable { } return inherited; } - private RankProfile resolveIndependentOfInheritance() { - for (RankProfile rankProfile : rankProfileRegistry.all()) { - if (rankProfile.getName().equals(inheritedName)) return rankProfile; - } - return null; - } + private String createFullyQualifiedName() { return (search != null) ? (search.getName() + "." + getName()) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index 46b785ccf42..ad85f68cb8a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -100,12 +100,6 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ remaining.forEach((name, rank) -> { if (areDependenciesReady(rank, rankProfileRegistry)) ready.add(rank); }); - if (ready.isEmpty() && ! deployProperties.featureFlags().enforceRankProfileInheritance()) { - // Dirty fallback to allow incorrect rankprofile inheritance to pass for now. - // We then handle one by one. - // TODO remove ASAP - ready.add(remaining.values().iterator().next()); - } processRankProfiles(ready, queryProfiles, importedModels, search, attributeFields, deployProperties, executor); ready.forEach(rank -> remaining.remove(rank.getName())); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java index 4332d8baea8..87fa74b92fe 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.indexinglanguage.ScriptParserContext; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; +import com.yahoo.yolean.Exceptions; /** * @author Einar M R Rosenvinge @@ -46,7 +47,7 @@ public class IndexingOperation implements FieldOperation { exp = new ScriptExpression(StatementExpression.newInstance(config)); } } catch (com.yahoo.vespa.indexinglanguage.parser.ParseException e) { - ParseException t = new ParseException("Error reported by IL parser: " + e.getMessage()); + ParseException t = new ParseException("Could not parse indexing statement: " + Exceptions.toMessageString(e)); t.initCause(e); throw t; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 937537e5f99..2c87fd5c5b3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -492,8 +492,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri boolean found = configProducer.cascadeConfig(builder); boolean foundOverride = configProducer.addUserConfig(builder); log.log(Level.FINE, () -> "Trying to get config for " + builder.getClass().getDeclaringClass().getName() + - " for config id " + quote(configProducer.getConfigId()) + - ", found=" + found + ", foundOverride=" + foundOverride); + " for config id " + quote(configProducer.getConfigId()) + + ", found=" + found + ", foundOverride=" + foundOverride); } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index ed1dc80d71d..af88d9c008a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -23,7 +23,7 @@ import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; -import com.yahoo.vespa.model.container.xml.PlatformBundles; +import com.yahoo.vespa.model.container.PlatformBundles; import java.util.Set; import java.util.TreeSet; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 0b2d0936235..14ead1cdece 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -38,7 +38,7 @@ import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; -import com.yahoo.vespa.model.container.xml.PlatformBundles; +import com.yahoo.vespa.model.container.PlatformBundles; import java.nio.file.Path; import java.util.Collections; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 5b1faa7218a..c91c4e92486 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -704,6 +704,10 @@ public class VespaMetricSet { metrics.add(new Metric("vds.idealstate.buckets_toomanycopies.average")); metrics.add(new Metric("vds.idealstate.buckets.average")); metrics.add(new Metric("vds.idealstate.buckets_notrusted.average")); + metrics.add(new Metric("vds.idealstate.bucket_replicas_moving_out.average")); + metrics.add(new Metric("vds.idealstate.bucket_replicas_copying_out.average")); + metrics.add(new Metric("vds.idealstate.bucket_replicas_copying_in.average")); + metrics.add(new Metric("vds.idealstate.bucket_replicas_syncing.average")); metrics.add(new Metric("vds.idealstate.delete_bucket.done_ok.rate")); metrics.add(new Metric("vds.idealstate.delete_bucket.done_failed.rate")); metrics.add(new Metric("vds.idealstate.delete_bucket.pending.average")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 1dd7b35cda5..5574082e334 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -34,7 +34,6 @@ import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.Servlet; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.configserver.ConfigserverCluster; -import com.yahoo.vespa.model.container.xml.PlatformBundles; import com.yahoo.vespa.model.utils.FileSender; import java.util.ArrayList; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index b915453b593..cdf8d592391 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -74,6 +74,8 @@ public abstract class Container extends AbstractService implements private final boolean retired; /** The unique index of this node */ private final int index; + private final boolean dumpHeapOnShutdownTimeout; + private final double shutdownTimeoutS; private final ComponentGroup<Handler<?>> handlers = new ComponentGroup<>(this, "handler"); private final ComponentGroup<Component<?, ?>> components = new ComponentGroup<>(this, "components"); @@ -90,6 +92,8 @@ public abstract class Container extends AbstractService implements this.parent = parent; this.retired = retired; this.index = index; + dumpHeapOnShutdownTimeout = deployState.featureFlags().containerDumpHeapOnShutdownTimeout(); + shutdownTimeoutS = deployState.featureFlags().containerShutdownTimeout(); this.defaultHttpServer = new JettyHttpServer("DefaultHttpServer", containerClusterOrNull(parent), deployState.isHosted()); if (getHttp() == null) { addChild(defaultHttpServer); @@ -315,7 +319,9 @@ public abstract class Container extends AbstractService implements .slobrokId(serviceSlobrokId())) .filedistributor(filedistributorConfig()) .discriminator((clusterName != null ? clusterName + "." : "" ) + name) - .nodeIndex(index); + .nodeIndex(index) + .shutdown.dumpHeapOnTimeout(dumpHeapOnShutdownTimeout) + .timeout(shutdownTimeoutS); } /** Returns the jvm args set explicitly for this node */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 532ac78f17e..f5b168958c0 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -171,6 +171,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> componentGroup = new ComponentGroup<>(this, "component"); + addCommonVespaBundles(); + addComponent(new StatisticsComponent()); addSimpleComponent(AccessLog.class); addComponent(new DefaultThreadpoolProvider(this, deployState.featureFlags().metricsproxyNumThreads())); @@ -459,6 +461,13 @@ public abstract class ContainerCluster<CONTAINER extends Container> } /** + * Adds the Vespa bundles that are necessary for all container types. + */ + public void addCommonVespaBundles() { + PlatformBundles.commonVespaBundles().forEach(this::addPlatformBundle); + } + + /** * Adds a bundle present at a known location at the target container nodes. * Note that the set of platform bundles cannot change during the jdisc container's lifetime. * diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index 65247f29281..25cb684932b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; -import com.yahoo.vespa.model.container.xml.PlatformBundles; import java.nio.file.Path; import java.util.List; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java new file mode 100644 index 00000000000..fad5d046f55 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java @@ -0,0 +1,134 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container; + +import com.yahoo.vespa.defaults.Defaults; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * @author gjoranv + * @author Ulf Lilleengen + */ +public class PlatformBundles { + + private enum JarSuffix { + JAR_WITH_DEPS("-jar-with-dependencies.jar"), + DEPLOY("-deploy.jar"); + + public final String suffix; + + JarSuffix(String suffix) { + this.suffix = suffix; + } + } + + public static final Path LIBRARY_PATH = Paths.get(Defaults.getDefaults().underVespaHome("lib/jars")); + public static final String searchAndDocprocBundle = "container-search-and-docproc"; + + public static Set<Path> commonVespaBundles() { + var bundles = new LinkedHashSet<Path>(); + commonVespaBundles.stream().map(PlatformBundles::absoluteBundlePath).forEach(bundles::add); + return Collections.unmodifiableSet(bundles); + } + + public static Path absoluteBundlePath(String fileName) { + return absoluteBundlePath(fileName, JarSuffix.JAR_WITH_DEPS); + } + + public static Path absoluteBundlePath(String fileName, JarSuffix jarSuffix) { + if (fileName == null) return null; + return LIBRARY_PATH.resolve(Paths.get(fileName + jarSuffix.suffix)); + } + + public static boolean isSearchAndDocprocClass(String className) { + return searchAndDocprocComponents.contains(className); + } + + // Bundles that must be loaded for all container types. + private static final List<String> commonVespaBundles = List.of( + "zkfacade", + "zookeeper-server" // TODO: not necessary in metrics-proxy. + ); + + // This is a hack to allow users to declare components from the search-and-docproc bundle without naming the bundle. + private static final Set<String> searchAndDocprocComponents = Set.of( + "com.yahoo.docproc.AbstractConcreteDocumentFactory", + "com.yahoo.docproc.DocumentProcessor", + "com.yahoo.docproc.SimpleDocumentProcessor", + "com.yahoo.docproc.util.JoinerDocumentProcessor", + "com.yahoo.docproc.util.SplitterDocumentProcessor", + "com.yahoo.example.TimingSearcher", + "com.yahoo.language.simple.SimpleLinguistics", + "com.yahoo.prelude.cluster.ClusterSearcher", + "com.yahoo.prelude.fastsearch.FastSearcher", + "com.yahoo.prelude.fastsearch.VespaBackEndSearcher", + "com.yahoo.prelude.querytransform.CJKSearcher", + "com.yahoo.prelude.querytransform.CollapsePhraseSearcher", + "com.yahoo.prelude.querytransform.LiteralBoostSearcher", + "com.yahoo.prelude.querytransform.NoRankingSearcher", + "com.yahoo.prelude.querytransform.NonPhrasingSearcher", + "com.yahoo.prelude.querytransform.NormalizingSearcher", + "com.yahoo.prelude.querytransform.PhrasingSearcher", + "com.yahoo.prelude.querytransform.RecallSearcher", + "com.yahoo.prelude.querytransform.StemmingSearcher", + "com.yahoo.prelude.searcher.BlendingSearcher", + "com.yahoo.prelude.searcher.FieldCollapsingSearcher", + "com.yahoo.prelude.searcher.FillSearcher", + "com.yahoo.prelude.searcher.JSONDebugSearcher", + "com.yahoo.prelude.searcher.JuniperSearcher", + "com.yahoo.prelude.searcher.MultipleResultsSearcher", + "com.yahoo.prelude.searcher.PosSearcher", + "com.yahoo.prelude.searcher.QuotingSearcher", + "com.yahoo.prelude.searcher.ValidateSortingSearcher", + "com.yahoo.prelude.semantics.SemanticSearcher", + "com.yahoo.prelude.statistics.StatisticsSearcher", + "com.yahoo.prelude.templates.SearchRendererAdaptor", + "com.yahoo.search.Searcher", + "com.yahoo.search.cluster.ClusterSearcher", + "com.yahoo.search.cluster.PingableSearcher", + "com.yahoo.search.federation.FederationSearcher", + "com.yahoo.search.federation.ForwardingSearcher", + "com.yahoo.search.federation.http.ConfiguredHTTPClientSearcher", + "com.yahoo.search.federation.http.ConfiguredHTTPProviderSearcher", + "com.yahoo.search.federation.http.HTTPClientSearcher", + "com.yahoo.search.federation.http.HTTPProviderSearcher", + "com.yahoo.search.federation.http.HTTPSearcher", + "com.yahoo.search.federation.news.NewsSearcher", + "com.yahoo.search.federation.vespa.VespaSearcher", + "com.yahoo.search.grouping.GroupingQueryParser", + "com.yahoo.search.grouping.GroupingValidator", + "com.yahoo.search.grouping.vespa.GroupingExecutor", + "com.yahoo.search.handler.SearchWithRendererHandler", + "com.yahoo.search.pagetemplates.PageTemplate", + "com.yahoo.search.pagetemplates.PageTemplateSearcher", + "com.yahoo.search.pagetemplates.engine.Resolver", + "com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver", + "com.yahoo.search.pagetemplates.engine.resolvers.RandomResolver", + "com.yahoo.search.pagetemplates.model.Renderer", + "com.yahoo.search.query.rewrite.QueryRewriteSearcher", + "com.yahoo.search.query.rewrite.SearchChainDispatcherSearcher", + "com.yahoo.search.query.rewrite.rewriters.GenericExpansionRewriter", + "com.yahoo.search.query.rewrite.rewriters.MisspellRewriter", + "com.yahoo.search.query.rewrite.rewriters.NameRewriter", + "com.yahoo.search.querytransform.AllLowercasingSearcher", + "com.yahoo.search.querytransform.DefaultPositionSearcher", + "com.yahoo.search.querytransform.LowercasingSearcher", + "com.yahoo.search.querytransform.NGramSearcher", + "com.yahoo.search.querytransform.VespaLowercasingSearcher", + "com.yahoo.search.rendering.Renderer", + "com.yahoo.search.rendering.SectionedRenderer", + "com.yahoo.search.searchchain.ForkingSearcher", + "com.yahoo.search.searchchain.example.ExampleSearcher", + "com.yahoo.search.searchers.CacheControlSearcher", + "com.yahoo.search.statistics.PeakQpsSearcher", + "com.yahoo.search.statistics.TimingSearcher", + "com.yahoo.vespa.streamingvisitors.MetricsSearcher", + "com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher" + ); + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Component.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Component.java index 6d891c55075..e7f6697aecc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Component.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Component.java @@ -69,4 +69,10 @@ public class Component<CHILD extends AbstractConfigProducer<?>, MODEL extends Co return getComponentId().compareTo(other.getComponentId()); } + @Override + public String toString() { + return "component " + getClassId() + + (getClassId().toString().equals(getComponentId().toString()) ? "" : ": " + getComponentId()); + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index ef2eaeb4654..fec8dd76102 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.ContainerSubsystem; -import com.yahoo.vespa.model.container.search.searchchain.LocalProvider; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; import com.yahoo.vespa.model.search.AbstractSearchCluster; import com.yahoo.vespa.model.search.IndexedSearchCluster; @@ -23,7 +22,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import static com.yahoo.vespa.model.container.xml.PlatformBundles.searchAndDocprocBundle; +import static com.yahoo.vespa.model.container.PlatformBundles.searchAndDocprocBundle; /** * @author gjoranv diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java index 232e8fcbd1a..832aede858e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java @@ -5,7 +5,7 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.model.container.component.Component; -import com.yahoo.vespa.model.container.xml.PlatformBundles; +import com.yahoo.vespa.model.container.PlatformBundles; import com.yahoo.vespa.model.search.IndexedSearchCluster; /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java index 248b30eafa7..8c45e5b013f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.model.container.search; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.component.Component; -import com.yahoo.vespa.model.container.xml.PlatformBundles; +import com.yahoo.vespa.model.container.PlatformBundles; public class RpcResourcePoolComponent extends Component<RpcResourcePoolComponent, ComponentModel> { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java index 12d74418f9f..ea0ad371e28 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.component.ComponentSpecification; +import com.yahoo.vespa.model.container.PlatformBundles; import org.w3c.dom.Element; import java.util.Arrays; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 4b45979c698..c318180fd56 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -59,6 +59,7 @@ import com.yahoo.vespa.model.container.ContainerModel; import com.yahoo.vespa.model.container.ContainerModelEvaluation; import com.yahoo.vespa.model.container.ContainerThreadpool; import com.yahoo.vespa.model.container.IdentityProvider; +import com.yahoo.vespa.model.container.PlatformBundles; import com.yahoo.vespa.model.container.SecretStore; import com.yahoo.vespa.model.container.component.AccessLogComponent; import com.yahoo.vespa.model.container.component.BindingPattern; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/PlatformBundles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/PlatformBundles.java deleted file mode 100644 index dc2437c1834..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/PlatformBundles.java +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.xml; - -import com.yahoo.vespa.defaults.Defaults; - -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Set; - -/** - * @author gjoranv - * @author Ulf Lilleengen - */ -public class PlatformBundles { - - private enum JarSuffix { - JAR_WITH_DEPS("-jar-with-dependencies.jar"), - DEPLOY("-deploy.jar"); - - public final String suffix; - - JarSuffix(String suffix) { - this.suffix = suffix; - } - } - - public static final Path LIBRARY_PATH = Paths.get(Defaults.getDefaults().underVespaHome("lib/jars")); - public static final String searchAndDocprocBundle = "container-search-and-docproc"; - - private static final Set<String> searchAndDocprocComponents; - - public static boolean isSearchAndDocprocClass(String className) { - return searchAndDocprocComponents.contains(className); - } - - public static Path absoluteBundlePath(String fileName) { - if (fileName == null) return null; - return LIBRARY_PATH.resolve(Paths.get(fileName + JarSuffix.JAR_WITH_DEPS.suffix)); - } - - // This is a hack to allow users to declare components from the search-and-docproc bundle without naming the bundle. - static { - searchAndDocprocComponents = Set.of( - "com.yahoo.docproc.AbstractConcreteDocumentFactory", - "com.yahoo.docproc.DocumentProcessor", - "com.yahoo.docproc.SimpleDocumentProcessor", - "com.yahoo.docproc.util.JoinerDocumentProcessor", - "com.yahoo.docproc.util.SplitterDocumentProcessor", - "com.yahoo.example.TimingSearcher", - "com.yahoo.language.simple.SimpleLinguistics", - "com.yahoo.prelude.cluster.ClusterSearcher", - "com.yahoo.prelude.fastsearch.FastSearcher", - "com.yahoo.prelude.fastsearch.VespaBackEndSearcher", - "com.yahoo.prelude.querytransform.CJKSearcher", - "com.yahoo.prelude.querytransform.CollapsePhraseSearcher", - "com.yahoo.prelude.querytransform.LiteralBoostSearcher", - "com.yahoo.prelude.querytransform.NoRankingSearcher", - "com.yahoo.prelude.querytransform.NonPhrasingSearcher", - "com.yahoo.prelude.querytransform.NormalizingSearcher", - "com.yahoo.prelude.querytransform.PhrasingSearcher", - "com.yahoo.prelude.querytransform.RecallSearcher", - "com.yahoo.prelude.querytransform.StemmingSearcher", - "com.yahoo.prelude.searcher.BlendingSearcher", - "com.yahoo.prelude.searcher.FieldCollapsingSearcher", - "com.yahoo.prelude.searcher.FillSearcher", - "com.yahoo.prelude.searcher.JSONDebugSearcher", - "com.yahoo.prelude.searcher.JuniperSearcher", - "com.yahoo.prelude.searcher.MultipleResultsSearcher", - "com.yahoo.prelude.searcher.PosSearcher", - "com.yahoo.prelude.searcher.QuotingSearcher", - "com.yahoo.prelude.searcher.ValidateSortingSearcher", - "com.yahoo.prelude.semantics.SemanticSearcher", - "com.yahoo.prelude.statistics.StatisticsSearcher", - "com.yahoo.prelude.templates.SearchRendererAdaptor", - "com.yahoo.search.Searcher", - "com.yahoo.search.cluster.ClusterSearcher", - "com.yahoo.search.cluster.PingableSearcher", - "com.yahoo.search.federation.FederationSearcher", - "com.yahoo.search.federation.ForwardingSearcher", - "com.yahoo.search.federation.http.ConfiguredHTTPClientSearcher", - "com.yahoo.search.federation.http.ConfiguredHTTPProviderSearcher", - "com.yahoo.search.federation.http.HTTPClientSearcher", - "com.yahoo.search.federation.http.HTTPProviderSearcher", - "com.yahoo.search.federation.http.HTTPSearcher", - "com.yahoo.search.federation.news.NewsSearcher", - "com.yahoo.search.federation.vespa.VespaSearcher", - "com.yahoo.search.grouping.GroupingQueryParser", - "com.yahoo.search.grouping.GroupingValidator", - "com.yahoo.search.grouping.vespa.GroupingExecutor", - "com.yahoo.search.handler.SearchWithRendererHandler", - "com.yahoo.search.pagetemplates.PageTemplate", - "com.yahoo.search.pagetemplates.PageTemplateSearcher", - "com.yahoo.search.pagetemplates.engine.Resolver", - "com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver", - "com.yahoo.search.pagetemplates.engine.resolvers.RandomResolver", - "com.yahoo.search.pagetemplates.model.Renderer", - "com.yahoo.search.query.rewrite.QueryRewriteSearcher", - "com.yahoo.search.query.rewrite.SearchChainDispatcherSearcher", - "com.yahoo.search.query.rewrite.rewriters.GenericExpansionRewriter", - "com.yahoo.search.query.rewrite.rewriters.MisspellRewriter", - "com.yahoo.search.query.rewrite.rewriters.NameRewriter", - "com.yahoo.search.querytransform.AllLowercasingSearcher", - "com.yahoo.search.querytransform.DefaultPositionSearcher", - "com.yahoo.search.querytransform.LowercasingSearcher", - "com.yahoo.search.querytransform.NGramSearcher", - "com.yahoo.search.querytransform.VespaLowercasingSearcher", - "com.yahoo.search.rendering.Renderer", - "com.yahoo.search.rendering.SectionedRenderer", - "com.yahoo.search.searchchain.ForkingSearcher", - "com.yahoo.search.searchchain.example.ExampleSearcher", - "com.yahoo.search.searchers.CacheControlSearcher", - "com.yahoo.search.statistics.PeakQpsSearcher", - "com.yahoo.search.statistics.TimingSearcher", - "com.yahoo.vespa.streamingvisitors.MetricsSearcher", - "com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher" - ); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java index de5eaa2278e..7d761eb07eb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java @@ -149,6 +149,7 @@ public class FileSender implements Serializable { String path = builder.getValue(); FileReference reference = sentFiles.get(path); if (reference == null) { + reference = fileRegistry.addFile(path); send(reference, services); sentFiles.put(path, reference); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java index 2c1f68e5ecc..aa068ec8f0e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java @@ -118,13 +118,8 @@ public class RankProfileTestCase extends SchemaTestCase { @Test public void requireThatSidewaysInheritanceIsImpossible() throws ParseException { - verifySidewaysInheritance(false); - verifySidewaysInheritance(true); - } - private void verifySidewaysInheritance(boolean enforce) throws ParseException { RankProfileRegistry registry = new RankProfileRegistry(); - SearchBuilder builder = new SearchBuilder(registry, setupQueryProfileTypes(), - new TestProperties().enforceRankProfileInheritance(enforce)); + SearchBuilder builder = new SearchBuilder(registry, setupQueryProfileTypes()); builder.importString(joinLines( "schema child1 {", " document child1 {", @@ -168,15 +163,8 @@ public class RankProfileTestCase extends SchemaTestCase { "}")); try { builder.build(true); - if (enforce) { - fail("Sideways inheritance should have been enforced"); - } else { - assertNotNull(builder.getSearch("child2")); - assertNotNull(builder.getSearch("child1")); - assertTrue(registry.get("child1", "child").inherits("parent")); - } + fail("Sideways inheritance should have been enforced"); } catch (IllegalArgumentException e) { - if (!enforce) fail("Sideways inheritance should have been allowed"); assertEquals("rank-profile 'child' inherits 'parent', but it does not exist anywhere in the inheritance of search 'child1'.", e.getMessage()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index 413daefdf75..4359e90e8a2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -51,8 +51,7 @@ public class MetricsProxyContainerClusterTest { var builder = new PlatformBundlesConfig.Builder(); model.getConfig(builder, CLUSTER_CONFIG_ID); PlatformBundlesConfig config = builder.build(); - assertEquals(1, config.bundlePaths().size()); - assertThat(config.bundlePaths(0), endsWith(METRICS_PROXY_BUNDLE_FILE.toString())); + assertThat(config.bundlePaths(), hasItem(endsWith(METRICS_PROXY_BUNDLE_FILE.toString()))); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index dd4f4c3b7d5..a66ea736a5b 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -38,8 +38,8 @@ import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; +import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasKey; -import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; @@ -383,10 +383,7 @@ public class ContainerClusterTest { cluster.getConfig(bundleBuilder); List<String> installedBundles = bundleBuilder.build().bundlePaths(); - assertEquals(expectedBundleNames.size(), installedBundles.size()); - assertThat(installedBundles, containsInAnyOrder( - expectedBundleNames.stream().map(CoreMatchers::endsWith).collect(Collectors.toList()) - )); + expectedBundleNames.forEach(b -> assertThat(installedBundles, hasItem(CoreMatchers.endsWith(b)))); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java index 70ae6a27324..4f715375e1f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java @@ -5,6 +5,7 @@ import com.yahoo.component.ComponentSpecification; import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.search.grouping.GroupingValidator; +import com.yahoo.vespa.model.container.PlatformBundles; import org.junit.Test; import org.w3c.dom.Element; import org.xml.sax.SAXException; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 2505aa3b01e..912ca23dce2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.TenantSecretStore; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; @@ -681,36 +682,47 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test public void qrconfig_is_produced() throws IOException, SAXException { + QrConfig qr = getQrConfig(new TestProperties()); + String hostname = HostName.getLocalhost(); // Using the same way of getting hostname as filedistribution model + assertEquals("default.container.0", qr.discriminator()); + assertEquals(19102, qr.rpc().port()); + assertEquals("vespa/service/default/container.0", qr.rpc().slobrokId()); + assertTrue(qr.rpc().enabled()); + assertEquals("", qr.rpc().host()); + assertFalse(qr.restartOnDeploy()); + assertEquals("filedistribution/" + hostname, qr.filedistributor().configid()); + assertEquals(50.0, qr.shutdown().timeout(), 0.00000000000001); + assertFalse(qr.shutdown().dumpHeapOnTimeout()); + } + private QrConfig getQrConfig(ModelContext.Properties properties) throws IOException, SAXException { String servicesXml = "<services>" + - "<admin version='3.0'>" + - " <nodes count='2'/>" + - "</admin>" + - "<container id ='default' version='1.0'>" + - " <nodes>" + - " <node hostalias='node1' />" + - " </nodes>" + - "</container>" + - "</services>"; + " <admin version='3.0'>" + + " <nodes count='2'/>" + + " </admin>" + + " <container id ='default' version='1.0'>" + + " <nodes>" + + " <node hostalias='node1' />" + + " </nodes>" + + " </container>" + + "</services>"; ApplicationPackage applicationPackage = new MockApplicationPackage.Builder() .withServices(servicesXml) .build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() .applicationPackage(applicationPackage) - .properties(new TestProperties()) + .properties(properties) .build()); - String hostname = HostName.getLocalhost(); // Using the same way of getting hostname as filedistribution model + return model.getConfig(QrConfig.class, "default/container.0"); + } - QrConfig config = model.getConfig(QrConfig.class, "default/container.0"); - assertEquals("default.container.0", config.discriminator()); - assertEquals(19102, config.rpc().port()); - assertEquals("vespa/service/default/container.0", config.rpc().slobrokId()); - assertTrue(config.rpc().enabled()); - assertEquals("", config.rpc().host()); - assertFalse(config.restartOnDeploy()); - assertEquals("filedistribution/" + hostname, config.filedistributor().configid()); + @Test + public void control_container_shutdown() throws IOException, SAXException { + QrConfig qr = getQrConfig(new TestProperties().containerShutdownTimeout(133).containerDumpHeapOnShutdownTimeout(true)); + assertEquals(133.0, qr.shutdown().timeout(), 0.00000000000001); + assertTrue(qr.shutdown().dumpHeapOnTimeout()); } @Test diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigVerification.java index 68a1d9b7333..56824c85413 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigVerification.java @@ -1,5 +1,5 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config; +package com.yahoo.vespa.config.proxy; import ai.vespa.util.http.hc5.VespaHttpClientBuilder; import com.yahoo.slime.ArrayTraverser; diff --git a/config-proxy/src/main/sh/vespa-config-verification.sh b/config-proxy/src/main/sh/vespa-config-verification.sh index 308d7f733a4..97201d772eb 100644 --- a/config-proxy/src/main/sh/vespa-config-verification.sh +++ b/config-proxy/src/main/sh/vespa-config-verification.sh @@ -79,4 +79,4 @@ export ROOT echo "# Using CLASSPATH=$CLASSPATH, args=$@" -java -cp $CLASSPATH:$ROOT/lib/jars/config-proxy-jar-with-dependencies.jar com.yahoo.vespa.config.ConfigVerification "$@" +java -cp $CLASSPATH:$ROOT/lib/jars/config-proxy-jar-with-dependencies.jar com.yahoo.vespa.config.proxy.ConfigVerification "$@" diff --git a/config/src/apps/vespa-get-config/getconfig.cpp b/config/src/apps/vespa-get-config/getconfig.cpp index 273a3abd1cd..679d118d0bc 100644 --- a/config/src/apps/vespa-get-config/getconfig.cpp +++ b/config/src/apps/vespa-get-config/getconfig.cpp @@ -240,7 +240,7 @@ GetConfig::Main() printf("defNamespace %s\n", rKey.getDefNamespace().c_str()); printf("configID %s\n", rKey.getConfigId().c_str()); - printf("configMD5 %s\n", rState.md5.c_str()); + printf("configXxhash64 %s\n", rState.xxhash64.c_str()); printf("generation %" PRId64 "\n", rState.generation); printf("trace %s\n", response->getTrace().toString().c_str()); diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java index 6d2ae3ef13e..7f81e937b3c 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java @@ -11,9 +11,12 @@ import static com.yahoo.vespa.config.ConfigPayloadApplier.IdentityPathAcquirer; /** * A utility class that can be used to transform config from one format to another. * - * @author Ulf Lilleengen, hmusum, Tony Vaagenes + * @author Ulf Lilleengen + * @author hmusum + * @author Tony Vaagenes */ public class ConfigTransformer<T extends ConfigInstance> { + /** * Workaround since FileAcquirer is in a separate module that depends on config. * Consider moving FileAcquirer into config instead. @@ -52,7 +55,7 @@ public class ConfigTransformer<T extends ConfigInstance> { /** * Create a ConfigBuilder from a payload, based on the <code>clazz</code> supplied. * - * @param payload a Payload to be transformed to builder. + * @param payload a Payload to be transformed to builder * @return a ConfigBuilder */ public ConfigInstance.Builder toConfigBuilder(ConfigPayload payload) { @@ -80,4 +83,5 @@ public class ConfigTransformer<T extends ConfigInstance> { return builder; } } + } diff --git a/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java b/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java index 123d7c22093..bc937ef57ea 100644 --- a/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java +++ b/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java @@ -5,7 +5,6 @@ import com.yahoo.config.ConfigBuilder; import com.yahoo.config.ConfigInstance; /** - * * A generic config with an internal generic builder that mimics a real config builder in order to support builders * when we don't have the schema. * diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java index 446ddf0560b..e6c9bc2175b 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.config.protocol; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; -import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.jrt.DataValue; import com.yahoo.jrt.Request; import com.yahoo.jrt.StringValue; @@ -11,7 +10,7 @@ import com.yahoo.jrt.Value; import com.yahoo.text.Utf8Array; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ErrorCode; -import com.yahoo.vespa.config.util.ConfigUtils; +import com.yahoo.vespa.config.PayloadChecksums; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -26,8 +25,7 @@ import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64; * The V3 config protocol implemented on the server side. The V3 protocol uses 2 fields: * * * A metadata field containing json data describing config generation, md5 and compression info - * * A data field containing compressed or uncompressed json config payload. This field can be empty if the payload - * has not changed since last request, triggering an optimization at the client where the previous payload is used instead. + * * A data field containing compressed or uncompressed json config payload * * The implementation of addOkResponse is optimized for doing as little copying of payload data as possible, ensuring * that we get a lower memory footprint. @@ -74,8 +72,6 @@ public class JRTServerConfigRequestV3 implements JRTServerConfigRequest { @Override public void addOkResponse(Payload payload, long generation, boolean applyOnRestart, PayloadChecksums payloadChecksums) { this.applyOnRestart = applyOnRestart; - boolean changedConfig = !payloadChecksums.equals(getRequestConfigChecksums()); - boolean changedConfigAndNewGeneration = changedConfig && ConfigUtils.isGenerationNewer(generation, getRequestGeneration()); Payload responsePayload = payload.withCompression(getCompressionType()); ByteArrayOutputStream byteArrayOutputStream = new NoCopyByteArrayOutputStream(4096); try { @@ -93,10 +89,6 @@ public class JRTServerConfigRequestV3 implements JRTServerConfigRequest { throw new RuntimeException("Payload is null for ' " + this + ", not able to create response"); } CompressionInfo compressionInfo = responsePayload.getCompressionInfo(); - // If payload is not being sent, we must adjust compression info to avoid client confusion. - if (!changedConfigAndNewGeneration) { - compressionInfo = CompressionInfo.create(compressionInfo.getCompressionType(), 0); - } compressionInfo.serialize(jsonGenerator); jsonGenerator.writeEndObject(); @@ -106,17 +98,13 @@ public class JRTServerConfigRequestV3 implements JRTServerConfigRequest { throw new IllegalArgumentException("Could not add OK response for " + this); } request.returnValues().add(createResponseValue(byteArrayOutputStream)); - if (changedConfigAndNewGeneration) { - ByteBuffer buf = responsePayload.getData().wrap(); - if (buf.hasArray() && buf.remaining() == buf.array().length) { - request.returnValues().add(new DataValue(buf.array())); - } else { - byte [] dst = new byte[buf.remaining()]; - buf.get(dst); - request.returnValues().add(new DataValue(dst)); - } + ByteBuffer buf = responsePayload.getData().wrap(); + if (buf.hasArray() && buf.remaining() == buf.array().length) { + request.returnValues().add(new DataValue(buf.array())); } else { - request.returnValues().add(new DataValue(new byte[0])); + byte[] dst = new byte[buf.remaining()]; + buf.get(dst); + request.returnValues().add(new DataValue(dst)); } } diff --git a/config/src/tests/configagent/configagent.cpp b/config/src/tests/configagent/configagent.cpp index d6766cce822..e4dc94fc2de 100644 --- a/config/src/tests/configagent/configagent.cpp +++ b/config/src/tests/configagent/configagent.cpp @@ -31,12 +31,12 @@ class MyConfigResponse : public ConfigResponse { public: MyConfigResponse(const ConfigKey & key, const ConfigValue & value, bool valid, int64_t timestamp, - const vespalib::string & md5, const std::string & errorMsg, int errorC0de, bool iserror) + const vespalib::string & xxhash64, const std::string & errorMsg, int errorC0de, bool iserror) : _key(key), _value(value), _fillCalled(false), _valid(valid), - _state(md5, timestamp, false), + _state(xxhash64, timestamp, false), _errorMessage(errorMsg), _errorCode(errorC0de), _isError(iserror) @@ -64,9 +64,9 @@ public: Trace _trace; - static ConfigResponse::UP createOKResponse(const ConfigKey & key, const ConfigValue & value, uint64_t timestamp = 10, const vespalib::string & md5 = "a") + static ConfigResponse::UP createOKResponse(const ConfigKey & key, const ConfigValue & value, uint64_t timestamp = 10, const vespalib::string & xxhash64 = "a") { - return std::make_unique<MyConfigResponse>(key, value, true, timestamp, md5, "", 0, false); + return std::make_unique<MyConfigResponse>(key, value, true, timestamp, xxhash64, "", 0, false); } static ConfigResponse::UP createServerErrorResponse(const ConfigKey & key, const ConfigValue & value) @@ -114,11 +114,11 @@ private: }; -ConfigValue createValue(const std::string & myField, const std::string & md5) +ConfigValue createValue(const std::string & myField, const std::string & xxhash64) { std::vector< vespalib::string > lines; lines.push_back("myField \"" + myField + "\""); - return ConfigValue(lines, md5); + return ConfigValue(lines, xxhash64); } static TimingValues testTimingValues( @@ -139,7 +139,7 @@ TEST("require that agent returns correct values") { ASSERT_EQUAL(500u, handler.getTimeout()); ASSERT_EQUAL(0u, handler.getWaitTime()); ConfigState cs; - ASSERT_EQUAL(cs.md5, handler.getConfigState().md5); + ASSERT_EQUAL(cs.xxhash64, handler.getConfigState().xxhash64); ASSERT_EQUAL(cs.generation, handler.getConfigState().generation); ASSERT_EQUAL(cs.applyOnRestart, handler.getConfigState().applyOnRestart); } @@ -167,7 +167,7 @@ TEST("require that important(the change) request is delivered to holder even if FRTConfigAgent handler(latch, testTimingValues); handler.handleResponse(MyConfigRequest(testKey), - MyConfigResponse::createOKResponse(testKey, testValue1, 1, testValue1.getMd5())); + MyConfigResponse::createOKResponse(testKey, testValue1, 1, testValue1.getXxhash64())); ASSERT_TRUE(latch->poll()); ConfigUpdate::UP update(latch->provide()); ASSERT_TRUE(update); @@ -176,9 +176,9 @@ TEST("require that important(the change) request is delivered to holder even if ASSERT_EQUAL("l33t", cfg.myField); handler.handleResponse(MyConfigRequest(testKey), - MyConfigResponse::createOKResponse(testKey, testValue2, 2, testValue2.getMd5())); + MyConfigResponse::createOKResponse(testKey, testValue2, 2, testValue2.getXxhash64())); handler.handleResponse(MyConfigRequest(testKey), - MyConfigResponse::createOKResponse(testKey, testValue2, 3, testValue2.getMd5())); + MyConfigResponse::createOKResponse(testKey, testValue2, 3, testValue2.getXxhash64())); ASSERT_TRUE(latch->poll()); update = latch->provide(); ASSERT_TRUE(update); diff --git a/config/src/tests/failover/failover.cpp b/config/src/tests/failover/failover.cpp index 2e039081716..cb8b7121b02 100644 --- a/config/src/tests/failover/failover.cpp +++ b/config/src/tests/failover/failover.cpp @@ -60,7 +60,7 @@ struct RPCServer : public FRT_Invokable { info.setString("uncompressedSize", "0"); root.setString(RESPONSE_CONFIGID, "myId"); root.setString(RESPONSE_CLIENT_HOSTNAME, "myhost"); - root.setString(RESPONSE_CONFIG_MD5, "md5"); + root.setString(RESPONSE_CONFIG_XXHASH64, "xxhash64"); root.setLong(RESPONSE_CONFIG_GENERATION, gen); root.setObject(RESPONSE_TRACE); Slime payload; diff --git a/config/src/tests/frt/frt.cpp b/config/src/tests/frt/frt.cpp index cb09b8f7254..5af02f22ed2 100644 --- a/config/src/tests/frt/frt.cpp +++ b/config/src/tests/frt/frt.cpp @@ -72,7 +72,7 @@ namespace { FRT_RPCRequest * createOKResponse(const vespalib::string & defName="", const vespalib::string & defMd5="", const vespalib::string & configId="", - const vespalib::string & configMd5="", + const vespalib::string & configXxhash64="", int changed=0, long generation=0, const std::vector<vespalib::string> & payload = std::vector<vespalib::string>(), @@ -85,7 +85,7 @@ namespace { ret.AddString(""); ret.AddString(defMd5.c_str()); ret.AddString(configId.c_str()); - ret.AddString(configMd5.c_str()); + ret.AddString(configXxhash64.c_str()); ret.AddInt32(changed); ret.AddInt64(generation); FRT_StringValue * payload_arr = ret.AddStringArray(payload.size()); @@ -268,16 +268,16 @@ TEST_FF("require that request is config task is scheduled", SourceFixture(), FRT TEST("require that v3 request is correctly initialized") { ConnectionMock conn; ConfigKey key = ConfigKey::create<MyConfig>("foobi"); - vespalib::string md5 = "mymd5"; + vespalib::string xxhash64 = "myxxhash64"; int64_t currentGeneration = 3; vespalib::string hostName = "myhost"; int64_t timeout = 3000; Trace traceIn(3); traceIn.trace(2, "Hei"); - FRTConfigRequestV3 v3req(&conn, key, md5, currentGeneration, hostName, + FRTConfigRequestV3 v3req(&conn, key, xxhash64, currentGeneration, hostName, timeout, traceIn, VespaVersion::fromString("1.2.3"), CompressionType::LZ4); - ASSERT_TRUE(v3req.verifyState(ConfigState(md5, 3, false))); - ASSERT_FALSE(v3req.verifyState(ConfigState(md5, 2, false))); + ASSERT_TRUE(v3req.verifyState(ConfigState(xxhash64, 3, false))); + ASSERT_FALSE(v3req.verifyState(ConfigState(xxhash64, 2, false))); ASSERT_FALSE(v3req.verifyState(ConfigState("xxx", 3, false))); ASSERT_FALSE(v3req.verifyState(ConfigState("xxx", 2, false))); @@ -297,7 +297,7 @@ TEST("require that v3 request is correctly initialized") { EXPECT_EQUAL(key.getConfigId(), root[REQUEST_CLIENT_CONFIGID].asString().make_string()); EXPECT_EQUAL(hostName, root[REQUEST_CLIENT_HOSTNAME].asString().make_string()); EXPECT_EQUAL(currentGeneration, root[REQUEST_CURRENT_GENERATION].asLong()); - EXPECT_EQUAL(md5, root[REQUEST_CONFIG_MD5].asString().make_string()); + EXPECT_EQUAL(xxhash64, root[REQUEST_CONFIG_XXHASH64].asString().make_string()); EXPECT_EQUAL(timeout, root[REQUEST_TIMEOUT].asLong()); EXPECT_EQUAL("LZ4", root[REQUEST_COMPRESSION_TYPE].asString().make_string()); EXPECT_EQUAL(root[REQUEST_VESPA_VERSION].asString().make_string(), "1.2.3"); @@ -322,7 +322,7 @@ struct V3RequestFixture { Cursor & root; FRT_RPCRequest * req; ConfigKey key; - vespalib::string md5; + vespalib::string xxhash64; int64_t generation; vespalib::string hostname; Trace traceIn; @@ -333,7 +333,7 @@ struct V3RequestFixture { root(slime.setObject()), req(conn.allocRPCRequest()), key(ConfigKey::create<BarConfig>("foobi")), - md5("mymd5"), + xxhash64("myxxhash64"), generation(3), hostname("myhhost"), traceIn(3) @@ -345,7 +345,7 @@ struct V3RequestFixture { root.setString(RESPONSE_DEF_MD5, Memory(key.getDefMd5())); root.setString(RESPONSE_CONFIGID, Memory(key.getConfigId())); root.setString(RESPONSE_CLIENT_HOSTNAME, Memory(hostname)); - root.setString(RESPONSE_CONFIG_MD5, Memory(md5)); + root.setString(RESPONSE_CONFIG_XXHASH64, Memory(xxhash64)); root.setLong(RESPONSE_CONFIG_GENERATION, generation); traceIn.serialize(root.setObject(RESPONSE_TRACE)); } @@ -379,7 +379,7 @@ struct V3RequestFixture { EXPECT_EQUAL(key.getConfigId(), responseKey.getConfigId()); EXPECT_EQUAL(hostname, response.getHostName()); ConfigState state(response.getConfigState()); - EXPECT_EQUAL(md5, state.md5); + EXPECT_EQUAL(xxhash64, state.xxhash64); EXPECT_EQUAL(generation, state.generation); ConfigValue value(response.getValue()); BarConfig::UP config(value.newInstance<BarConfig>()); diff --git a/config/src/tests/misc/misc.cpp b/config/src/tests/misc/misc.cpp index 1a8b8a59ede..55040242732 100644 --- a/config/src/tests/misc/misc.cpp +++ b/config/src/tests/misc/misc.cpp @@ -15,11 +15,11 @@ TEST("requireThatConfigUpdateWorks") { std::vector<vespalib::string> lines; lines.push_back("foo"); - ConfigUpdate up(ConfigValue(lines, "mymd5"), true, 1337); + ConfigUpdate up(ConfigValue(lines, "myxxhash"), true, 1337); ASSERT_EQUAL(1337, up.getGeneration()); ASSERT_TRUE(up.hasChanged()); - ConfigUpdate up2(ConfigValue(lines, "mymd52"), false, 1338); + ConfigUpdate up2(ConfigValue(lines, "myxxhash2"), false, 1338); ASSERT_EQUAL(1338, up2.getGeneration()); ASSERT_FALSE(up2.hasChanged()); } @@ -27,22 +27,22 @@ TEST("requireThatConfigUpdateWorks") { TEST("requireThatConfigValueWorks") { std::vector<vespalib::string> lines; lines.push_back("myFooField \"bar\""); - ConfigValue v1(lines, calculateContentMd5(lines)); - ConfigValue v2(lines, calculateContentMd5(lines)); - ConfigValue v3(lines, calculateContentMd5(lines)); + ConfigValue v1(lines, calculateContentXxhash64(lines)); + ConfigValue v2(lines, calculateContentXxhash64(lines)); + ConfigValue v3(lines, calculateContentXxhash64(lines)); lines.push_back("myFooField \"bar2\""); - ConfigValue v4(lines, calculateContentMd5(lines)); + ConfigValue v4(lines, calculateContentXxhash64(lines)); ASSERT_TRUE(v1 == v2); ASSERT_TRUE(v1 == v3); } TEST("requireThatConfigKeyWorks") { - ConfigKey key1("id1", "def1", "namespace1", "md51"); - ConfigKey key2("id1", "def1", "namespace1", "md51"); - ConfigKey key3("id2", "def1", "namespace1", "md51"); - ConfigKey key4("id1", "def2", "namespace1", "md51"); - ConfigKey key5("id1", "def1", "namespace2", "md51"); - ConfigKey key6("id1", "def1", "namespace1", "md52"); // Special case. Md5 does not matter, so should be qual to key1 and key2 + ConfigKey key1("id1", "def1", "namespace1", "xxhash1"); + ConfigKey key2("id1", "def1", "namespace1", "xxhash1"); + ConfigKey key3("id2", "def1", "namespace1", "xxhash1"); + ConfigKey key4("id1", "def2", "namespace1", "xxhash1"); + ConfigKey key5("id1", "def1", "namespace2", "xxhash1"); + ConfigKey key6("id1", "def1", "namespace1", "xxhash2"); // Special case. xxhash64 does not matter, so should be qual to key1 and key2 ASSERT_TRUE(key1 == key2); @@ -111,7 +111,7 @@ TEST("require that config key initializes schema") std::vector<vespalib::string> schema; schema.push_back("foo"); schema.push_back("bar"); - ConfigKey key("id1", "def1", "namespace1", "md51", schema); + ConfigKey key("id1", "def1", "namespace1", "xxhash1", schema); const std::vector<vespalib::string> &vref(key.getDefSchema()); for (size_t i = 0; i < schema.size(); i++) { ASSERT_EQUAL(schema[i], vref[i]); @@ -131,6 +131,7 @@ TEST("require that error codes are correctly translated to strings") { ASSERT_CONFIG(ILLEGAL_CONFIGID); ASSERT_CONFIG(ILLEGAL_DEF_MD5); ASSERT_CONFIG(ILLEGAL_CONFIG_MD5); + ASSERT_CONFIG(ILLEGAL_CONFIG_MD5); ASSERT_CONFIG(ILLEGAL_TIMEOUT); ASSERT_CONFIG(ILLEGAL_TIMESTAMP); ASSERT_CONFIG(ILLEGAL_NAME_SPACE); diff --git a/config/src/tests/subscriber/subscriber.cpp b/config/src/tests/subscriber/subscriber.cpp index d58699f26e0..5c5714a867a 100644 --- a/config/src/tests/subscriber/subscriber.cpp +++ b/config/src/tests/subscriber/subscriber.cpp @@ -20,7 +20,7 @@ namespace { { std::vector< vespalib::string > lines; lines.push_back(value); - return ConfigValue(lines, calculateContentMd5(lines)); + return ConfigValue(lines, calculateContentXxhash64(lines)); } ConfigValue createFooValue(const std::string & value) diff --git a/config/src/vespa/config/common/configstate.h b/config/src/vespa/config/common/configstate.h index 2dbea3cc30f..143d77e4ab9 100644 --- a/config/src/vespa/config/common/configstate.h +++ b/config/src/vespa/config/common/configstate.h @@ -13,17 +13,17 @@ struct ConfigState { public: ConfigState() - : md5(""), + : xxhash64(""), generation(0), applyOnRestart(false) { } - ConfigState(const vespalib::string & md5sum, int64_t gen, bool _applyOnRestart) - : md5(md5sum), + ConfigState(const vespalib::string & xxhash, int64_t gen, bool _applyOnRestart) + : xxhash64(xxhash), generation(gen), applyOnRestart(_applyOnRestart) { } - vespalib::string md5; + vespalib::string xxhash64; int64_t generation; bool applyOnRestart; @@ -32,7 +32,7 @@ public: } bool hasDifferentPayloadFrom(const ConfigState & other) const { - return (md5.compare(other.md5) != 0); + return (xxhash64.compare(other.xxhash64) != 0); } }; diff --git a/config/src/vespa/config/common/configvalue.cpp b/config/src/vespa/config/common/configvalue.cpp index d5c0c2047df..ce6fd3f20da 100644 --- a/config/src/vespa/config/common/configvalue.cpp +++ b/config/src/vespa/config/common/configvalue.cpp @@ -6,22 +6,22 @@ namespace config { -ConfigValue::ConfigValue(const std::vector<vespalib::string> & lines, const vespalib::string & md5sum) +ConfigValue::ConfigValue(const std::vector<vespalib::string> & lines, const vespalib::string & xxhash) : _payload(), _lines(lines), - _md5sum(md5sum) + _xxhash64(xxhash) { } ConfigValue::ConfigValue() : _payload(), _lines(), - _md5sum() + _xxhash64() { } -ConfigValue::ConfigValue(PayloadPtr payload, const vespalib::string & md5) +ConfigValue::ConfigValue(PayloadPtr payload, const vespalib::string & xxhash) : _payload(std::move(payload)), _lines(), - _md5sum(md5) + _xxhash64(xxhash) { } ConfigValue::ConfigValue(const ConfigValue &) = default; @@ -32,7 +32,7 @@ ConfigValue::~ConfigValue() = default; int ConfigValue::operator==(const ConfigValue & rhs) const { - return (_md5sum.compare(rhs._md5sum) == 0); + return (_xxhash64.compare(rhs._xxhash64) == 0); } int diff --git a/config/src/vespa/config/common/configvalue.h b/config/src/vespa/config/common/configvalue.h index a0450328f30..f5bfae00c19 100644 --- a/config/src/vespa/config/common/configvalue.h +++ b/config/src/vespa/config/common/configvalue.h @@ -21,8 +21,8 @@ typedef std::shared_ptr<const protocol::Payload> PayloadPtr; class ConfigValue { public: typedef std::unique_ptr<ConfigValue> UP; - ConfigValue(const std::vector<vespalib::string> & lines, const vespalib::string & md5sum); - ConfigValue(PayloadPtr data, const vespalib::string & md5sum); + ConfigValue(const std::vector<vespalib::string> & lines, const vespalib::string & xxhash); + ConfigValue(PayloadPtr data, const vespalib::string & xxhash); ConfigValue(); ConfigValue(const ConfigValue &); ConfigValue & operator = (const ConfigValue &); @@ -36,7 +36,7 @@ public: const std::vector<vespalib::string> & getLines() const { return _lines; } std::vector<vespalib::string> getLegacyFormat() const; const vespalib::string asJson() const; - const vespalib::string getMd5() const { return _md5sum; } + const vespalib::string getXxhash64() const { return _xxhash64; } void serializeV1(::vespalib::slime::Cursor & cursor) const; void serializeV2(::vespalib::slime::Cursor & cursor) const; @@ -47,7 +47,7 @@ public: private: PayloadPtr _payload; std::vector<vespalib::string> _lines; - vespalib::string _md5sum; + vespalib::string _xxhash64; }; } //namespace config diff --git a/config/src/vespa/config/common/misc.cpp b/config/src/vespa/config/common/misc.cpp index 1040962e25c..e1b4390caf5 100644 --- a/config/src/vespa/config/common/misc.cpp +++ b/config/src/vespa/config/common/misc.cpp @@ -1,7 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "misc.h" -#include <vespa/vespalib/util/md5.h> +#include <iostream> +#include <sstream> +#include <xxhash.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/data/slime/slime.h> @@ -12,12 +14,12 @@ using vespalib::Memory; namespace config { vespalib::string -calculateContentMd5(const std::vector<vespalib::string> & fileContents) +calculateContentXxhash64(const std::vector<vespalib::string> & fileContents) { vespalib::string normalizedLines; - int compact_md5size = 16; - unsigned char md5sum[compact_md5size]; + XXH64_hash_t xxhash64; vespalib::asciistream s; + std::stringstream ss; // remove comments, trailing spaces and empty lines // TODO: Remove multiple spaces and space before comma, like in Java @@ -30,16 +32,11 @@ calculateContentMd5(const std::vector<vespalib::string> & fileContents) normalizedLines += line; } } - fastc_md5sum((const unsigned char*)normalizedLines.c_str(), normalizedLines.size(), md5sum); + xxhash64 = XXH64((const unsigned char*)normalizedLines.c_str(), normalizedLines.size(), 0); - // convert to 32 character hex string - for (int i = 0; i < compact_md5size; i++) { - if (md5sum[i] < 16) { - s << "0"; - } - s << vespalib::hex << (int)md5sum[i]; - } - return s.str(); + ss << std::hex << xxhash64; + ss << std::endl; + return ss.str(); } bool diff --git a/config/src/vespa/config/common/misc.h b/config/src/vespa/config/common/misc.h index 0299ef001f1..089fd890224 100644 --- a/config/src/vespa/config/common/misc.h +++ b/config/src/vespa/config/common/misc.h @@ -19,7 +19,7 @@ namespace config { /** * Miscellaneous utility functions specific to config. */ -vespalib::string calculateContentMd5(const std::vector<vespalib::string> & fileContents); +vespalib::string calculateContentXxhash64(const std::vector<vespalib::string> & fileContents); bool isGenerationNewer(int64_t newGen, int64_t oldGen); diff --git a/config/src/vespa/config/file/filesource.cpp b/config/src/vespa/config/file/filesource.cpp index fcc69f68066..8e25d5ccffc 100644 --- a/config/src/vespa/config/file/filesource.cpp +++ b/config/src/vespa/config/file/filesource.cpp @@ -27,10 +27,10 @@ FileSource::getConfig() int64_t last = getLast(_fileName); if (last > _lastLoaded) { - _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, calculateContentMd5(lines)), true, _generation))); + _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, calculateContentXxhash64(lines)), true, _generation))); _lastLoaded = last; } else { - _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, calculateContentMd5(lines)), false, _generation))); + _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, calculateContentXxhash64(lines)), false, _generation))); } } diff --git a/config/src/vespa/config/frt/frtconfigagent.cpp b/config/src/vespa/config/frt/frtconfigagent.cpp index 0e5ee4f6b05..30112b677e1 100644 --- a/config/src/vespa/config/frt/frtconfigagent.cpp +++ b/config/src/vespa/config/frt/frtconfigagent.cpp @@ -27,7 +27,7 @@ FRTConfigAgent::handleResponse(const ConfigRequest & request, ConfigResponse::UP { if (LOG_WOULD_LOG(spam)) { const ConfigKey & key(request.getKey()); - LOG(spam, "current state for %s: generation %" PRId64 " md5 %s", key.toString().c_str(), _configState.generation, _configState.md5.c_str()); + LOG(spam, "current state for %s: generation %" PRId64 " xxhash64 %s", key.toString().c_str(), _configState.generation, _configState.xxhash64.c_str()); } if (response->validateResponse() && !response->isError()) { handleOKResponse(request, std::move(response)); @@ -57,12 +57,12 @@ void FRTConfigAgent::handleUpdatedGeneration(const ConfigKey & key, const ConfigState & newState, const ConfigValue & configValue) { if (LOG_WOULD_LOG(spam)) { - LOG(spam, "new generation %" PRId64 " md5:%s for key %s", newState.generation, newState.md5.c_str(), key.toString().c_str()); - LOG(spam, "Old config: md5:%s \n%s", _latest.getMd5().c_str(), _latest.asJson().c_str()); - LOG(spam, "New config: md5:%s \n%s", configValue.getMd5().c_str(), configValue.asJson().c_str()); + LOG(spam, "new generation %" PRId64 " xxhash64:%s for key %s", newState.generation, newState.xxhash64.c_str(), key.toString().c_str()); + LOG(spam, "Old config: xxhash64:%s \n%s", _latest.getXxhash64().c_str(), _latest.asJson().c_str()); + LOG(spam, "New config: xxhash64:%s \n%s", configValue.getXxhash64().c_str(), configValue.asJson().c_str()); } bool changed = false; - if (_latest.getMd5() != configValue.getMd5()) { + if (_latest.getXxhash64() != configValue.getXxhash64()) { _latest = configValue; changed = true; } diff --git a/config/src/vespa/config/frt/frtconfigrequestfactory.cpp b/config/src/vespa/config/frt/frtconfigrequestfactory.cpp index fbc13556d14..3573b8c9cfe 100644 --- a/config/src/vespa/config/frt/frtconfigrequestfactory.cpp +++ b/config/src/vespa/config/frt/frtconfigrequestfactory.cpp @@ -24,7 +24,7 @@ FRTConfigRequest::UP FRTConfigRequestFactory::createConfigRequest(const ConfigKey & key, Connection * connection, const ConfigState & state, int64_t serverTimeout) const { - return make_unique<FRTConfigRequestV3>(connection, key, state.md5, state.generation, _hostName, + return make_unique<FRTConfigRequestV3>(connection, key, state.xxhash64, state.generation, _hostName, serverTimeout, Trace(_traceLevel), _vespaVersion, _compressionType); } diff --git a/config/src/vespa/config/frt/frtconfigresponsev3.cpp b/config/src/vespa/config/frt/frtconfigresponsev3.cpp index 80cdf88a79a..351f0fc8136 100644 --- a/config/src/vespa/config/frt/frtconfigresponsev3.cpp +++ b/config/src/vespa/config/frt/frtconfigresponsev3.cpp @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "frtconfigresponsev3.h" #include "compressioninfo.h" #include <vespa/fnet/frt/values.h> @@ -53,7 +53,7 @@ FRTConfigResponseV3::getResponseTypes() const ConfigValue FRTConfigResponseV3::readConfigValue() const { - vespalib::string md5(_data->get()[RESPONSE_CONFIG_MD5].asString().make_string()); + vespalib::string xxhash64(_data->get()[RESPONSE_CONFIG_XXHASH64].asString().make_string()); CompressionInfo info; info.deserialize(_data->get()[RESPONSE_COMPRESSION_INFO]); auto slime = std::make_unique<Slime>(); @@ -67,9 +67,9 @@ FRTConfigResponseV3::readConfigValue() const } } if (LOG_WOULD_LOG(spam)) { - LOG(spam, "read config value md5(%s), payload size: %lu", md5.c_str(), data.memRef.size); + LOG(spam, "read config value xxhash64(%s), payload size: %lu", xxhash64.c_str(), data.memRef.size); } - return ConfigValue(std::make_shared<V3Payload>(std::move(slime)), md5); + return ConfigValue(std::make_shared<V3Payload>(std::move(slime)), xxhash64); } } // namespace config diff --git a/config/src/vespa/config/frt/protocol.cpp b/config/src/vespa/config/frt/protocol.cpp index 4a1b0a6ddef..c6c99ccf1ff 100644 --- a/config/src/vespa/config/frt/protocol.cpp +++ b/config/src/vespa/config/frt/protocol.cpp @@ -25,6 +25,7 @@ const Memory REQUEST_DEF_CONTENT = "defContent"; const Memory REQUEST_CLIENT_CONFIGID = "configId"; const Memory REQUEST_CLIENT_HOSTNAME = "clientHostname"; const Memory REQUEST_CONFIG_MD5 = "configMD5"; +const Memory REQUEST_CONFIG_XXHASH64 = "configXxhash64"; const Memory REQUEST_CURRENT_GENERATION = "currentGeneration"; const Memory REQUEST_TIMEOUT = "timeout"; const Memory REQUEST_TRACE = "trace"; @@ -36,7 +37,7 @@ const Memory RESPONSE_DEF_NAMESPACE = "defNamespace"; const Memory RESPONSE_DEF_MD5 = "defMD5"; const Memory RESPONSE_CONFIGID = "configId"; const Memory RESPONSE_CLIENT_HOSTNAME = "clientHostname"; -const Memory RESPONSE_CONFIG_MD5 = "configMD5"; +const Memory RESPONSE_CONFIG_XXHASH64 = "configXxhash64"; const Memory RESPONSE_CONFIG_GENERATION = "generation"; const Memory RESPONSE_PAYLOAD = "payload"; const Memory RESPONSE_TRACE = "trace"; diff --git a/config/src/vespa/config/frt/protocol.h b/config/src/vespa/config/frt/protocol.h index 0ec16952701..ce09217b619 100644 --- a/config/src/vespa/config/frt/protocol.h +++ b/config/src/vespa/config/frt/protocol.h @@ -36,7 +36,7 @@ extern const vespalib::Memory REQUEST_DEF_MD5; extern const vespalib::Memory REQUEST_DEF_CONTENT; extern const vespalib::Memory REQUEST_CLIENT_CONFIGID; extern const vespalib::Memory REQUEST_CLIENT_HOSTNAME; -extern const vespalib::Memory REQUEST_CONFIG_MD5; +extern const vespalib::Memory REQUEST_CONFIG_XXHASH64; extern const vespalib::Memory REQUEST_CURRENT_GENERATION; extern const vespalib::Memory REQUEST_TIMEOUT; extern const vespalib::Memory REQUEST_TRACE; @@ -48,7 +48,7 @@ extern const vespalib::Memory RESPONSE_DEF_NAMESPACE; extern const vespalib::Memory RESPONSE_DEF_MD5; extern const vespalib::Memory RESPONSE_CONFIGID; extern const vespalib::Memory RESPONSE_CLIENT_HOSTNAME; -extern const vespalib::Memory RESPONSE_CONFIG_MD5; +extern const vespalib::Memory RESPONSE_CONFIG_XXHASH64; extern const vespalib::Memory RESPONSE_CONFIG_GENERATION; extern const vespalib::Memory RESPONSE_PAYLOAD; extern const vespalib::Memory RESPONSE_TRACE; diff --git a/config/src/vespa/config/frt/slimeconfigrequest.cpp b/config/src/vespa/config/frt/slimeconfigrequest.cpp index 8a6706974f6..3573f8c07b9 100644 --- a/config/src/vespa/config/frt/slimeconfigrequest.cpp +++ b/config/src/vespa/config/frt/slimeconfigrequest.cpp @@ -23,7 +23,7 @@ const vespalib::string SlimeConfigRequest::REQUEST_TYPES = "s"; SlimeConfigRequest::SlimeConfigRequest(Connection * connection, const ConfigKey & key, - const vespalib::string & configMd5, + const vespalib::string & configXxhash64, int64_t currentGeneration, const vespalib::string & hostName, int64_t serverTimeout, @@ -35,7 +35,7 @@ SlimeConfigRequest::SlimeConfigRequest(Connection * connection, : FRTConfigRequest(connection, key), _data() { - populateSlimeRequest(key, configMd5, currentGeneration, hostName, serverTimeout, trace, vespaVersion, protocolVersion, compressionType); + populateSlimeRequest(key, configXxhash64, currentGeneration, hostName, serverTimeout, trace, vespaVersion, protocolVersion, compressionType); _request->SetMethodName(methodName.c_str()); _parameters.AddString(createJsonFromSlime(_data).c_str()); } @@ -43,13 +43,13 @@ SlimeConfigRequest::SlimeConfigRequest(Connection * connection, bool SlimeConfigRequest::verifyState(const ConfigState & state) const { - return (state.md5.compare(_data[REQUEST_CONFIG_MD5].asString().make_stringref()) == 0 && + return (state.xxhash64.compare(_data[REQUEST_CONFIG_XXHASH64].asString().make_stringref()) == 0 && state.generation == _data[REQUEST_CURRENT_GENERATION].asLong()); } void SlimeConfigRequest::populateSlimeRequest(const ConfigKey & key, - const vespalib::string & configMd5, + const vespalib::string & configXxhash64, int64_t currentGeneration, const vespalib::string & hostName, int64_t serverTimeout, @@ -67,7 +67,7 @@ SlimeConfigRequest::populateSlimeRequest(const ConfigKey & key, def.serialize(root.setArray(REQUEST_DEF_CONTENT)); root.setString(REQUEST_CLIENT_CONFIGID, Memory(key.getConfigId())); root.setString(REQUEST_CLIENT_HOSTNAME, Memory(hostName)); - root.setString(REQUEST_CONFIG_MD5, Memory(configMd5)); + root.setString(REQUEST_CONFIG_XXHASH64, Memory(configXxhash64)); root.setLong(REQUEST_CURRENT_GENERATION, currentGeneration); root.setLong(REQUEST_TIMEOUT, serverTimeout); trace.serialize(root.setObject(REQUEST_TRACE)); diff --git a/config/src/vespa/config/frt/slimeconfigrequest.h b/config/src/vespa/config/frt/slimeconfigrequest.h index 6f2f42c98d5..d25530f815c 100644 --- a/config/src/vespa/config/frt/slimeconfigrequest.h +++ b/config/src/vespa/config/frt/slimeconfigrequest.h @@ -19,7 +19,7 @@ class SlimeConfigRequest : public FRTConfigRequest { public: SlimeConfigRequest(Connection * connection, const ConfigKey & key, - const vespalib::string & configMd5, + const vespalib::string & configXxhash64, int64_t currentGeneration, const vespalib::string & hostName, int64_t serverTimeout, @@ -33,7 +33,7 @@ public: virtual ConfigResponse::UP createResponse(FRT_RPCRequest * request) const override = 0; private: void populateSlimeRequest(const ConfigKey & key, - const vespalib::string & configMd5, + const vespalib::string & configXxhash64, int64_t currentGeneration, const vespalib::string & hostName, int64_t serverTimeout, diff --git a/config/src/vespa/config/frt/slimeconfigresponse.cpp b/config/src/vespa/config/frt/slimeconfigresponse.cpp index af224008d01..c1f3df7f674 100644 --- a/config/src/vespa/config/frt/slimeconfigresponse.cpp +++ b/config/src/vespa/config/frt/slimeconfigresponse.cpp @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "slimeconfigresponse.h" #include <vespa/config/common/misc.h> #include <vespa/fnet/frt/values.h> @@ -66,7 +66,7 @@ ConfigState SlimeConfigResponse::readState() const { const Slime & data(*_data); - return ConfigState(data.get()[RESPONSE_CONFIG_MD5].asString().make_string(), + return ConfigState(data.get()[RESPONSE_CONFIG_XXHASH64].asString().make_string(), data.get()[RESPONSE_CONFIG_GENERATION].asLong(), data.get()[RESPONSE_APPLY_ON_RESTART].asBool()); } diff --git a/config/src/vespa/config/print/asciiconfigreader.hpp b/config/src/vespa/config/print/asciiconfigreader.hpp index b4a25f4fbbd..9d142e83c00 100644 --- a/config/src/vespa/config/print/asciiconfigreader.hpp +++ b/config/src/vespa/config/print/asciiconfigreader.hpp @@ -27,7 +27,7 @@ AsciiConfigReader<ConfigType>::read() while (getline(_is, line)) { lines.push_back(line); } - return std::unique_ptr<ConfigType>(new ConfigType(ConfigValue(lines, calculateContentMd5(lines)))); + return std::unique_ptr<ConfigType>(new ConfigType(ConfigValue(lines, calculateContentXxhash64(lines)))); } } // namespace config diff --git a/config/src/vespa/config/print/fileconfigreader.hpp b/config/src/vespa/config/print/fileconfigreader.hpp index bfebb4b89b9..87a4e20866c 100644 --- a/config/src/vespa/config/print/fileconfigreader.hpp +++ b/config/src/vespa/config/print/fileconfigreader.hpp @@ -44,7 +44,7 @@ FileConfigReader<ConfigType>::read() for (std::getline(f, line); f; std::getline(f, line)) { lines.push_back(line); } - return std::unique_ptr<ConfigType>(new ConfigType(ConfigValue(lines, calculateContentMd5(lines)))); + return std::unique_ptr<ConfigType>(new ConfigType(ConfigValue(lines, calculateContentXxhash64(lines)))); } } // namespace config diff --git a/config/src/vespa/config/print/istreamconfigreader.hpp b/config/src/vespa/config/print/istreamconfigreader.hpp index 70f7b7e5f28..3a501d26245 100644 --- a/config/src/vespa/config/print/istreamconfigreader.hpp +++ b/config/src/vespa/config/print/istreamconfigreader.hpp @@ -31,7 +31,7 @@ IstreamConfigReader<ConfigType>::read() while (getline(_is, line)) { lines.push_back(line); } - return std::unique_ptr<ConfigType>(new ConfigType(ConfigValue(lines, calculateContentMd5(lines)))); + return std::unique_ptr<ConfigType>(new ConfigType(ConfigValue(lines, calculateContentXxhash64(lines)))); } } // namespace config diff --git a/config/src/vespa/config/raw/rawsource.cpp b/config/src/vespa/config/raw/rawsource.cpp index 2b38cff214c..a6c4da0b4dd 100644 --- a/config/src/vespa/config/raw/rawsource.cpp +++ b/config/src/vespa/config/raw/rawsource.cpp @@ -16,7 +16,7 @@ void RawSource::getConfig() { auto lines(readConfig()); - ConfigValue value(lines, calculateContentMd5(lines)); + ConfigValue value(lines, calculateContentXxhash64(lines)); _holder->handle(ConfigUpdate::UP(new ConfigUpdate(value, true, 1))); } diff --git a/config/src/vespa/config/retriever/configsnapshot.cpp b/config/src/vespa/config/retriever/configsnapshot.cpp index 7aaf4ffd6f8..35f4415a26e 100644 --- a/config/src/vespa/config/retriever/configsnapshot.cpp +++ b/config/src/vespa/config/retriever/configsnapshot.cpp @@ -150,7 +150,7 @@ void ConfigSnapshot::serializeValueV2(Cursor & cursor, const Value & value) const { cursor.setDouble("lastChanged", value.first); - cursor.setString("md5", Memory(value.second.getMd5())); + cursor.setString("xxhash64", Memory(value.second.getXxhash64())); value.second.serializeV2(cursor.setObject("payload")); } @@ -225,7 +225,7 @@ ConfigSnapshot::deserializeValueV1(Inspector & inspector) const for (size_t i = 0; i < s.children(); i++) { payload.push_back(s[i].asString().make_string()); } - return Value(lastChanged, ConfigValue(payload, calculateContentMd5(payload))); + return Value(lastChanged, ConfigValue(payload, calculateContentXxhash64(payload))); } namespace { @@ -247,10 +247,10 @@ std::pair<int64_t, ConfigValue> ConfigSnapshot::deserializeValueV2(Inspector & inspector) const { int64_t lastChanged = static_cast<int64_t>(inspector["lastChanged"].asDouble()); - vespalib::string md5(inspector["md5"].asString().make_string()); + vespalib::string xxhash64(inspector["xxhash64"].asString().make_string()); auto payload = std::make_unique<FixedPayload>(); copySlimeObject(inspector["payload"], payload->getData().setObject()); - return Value(lastChanged, ConfigValue(std::move(payload) , md5)); + return Value(lastChanged, ConfigValue(std::move(payload), xxhash64)); } } diff --git a/config/src/vespa/config/set/configinstancesourcefactory.cpp b/config/src/vespa/config/set/configinstancesourcefactory.cpp index 7f963847997..5ebffd4c4e9 100644 --- a/config/src/vespa/config/set/configinstancesourcefactory.cpp +++ b/config/src/vespa/config/set/configinstancesourcefactory.cpp @@ -14,8 +14,8 @@ public: void close() override { } void getConfig() override { std::vector<vespalib::string> lines(_buffer.getlines()); - std::string currentMd5(config::calculateContentMd5(lines)); - _holder->handle(config::ConfigUpdate::UP(new config::ConfigUpdate(config::ConfigValue(lines, currentMd5), true, _generation))); + std::string currentXxhash64(config::calculateContentXxhash64(lines)); + _holder->handle(config::ConfigUpdate::UP(new config::ConfigUpdate(config::ConfigValue(lines, currentXxhash64), true, _generation))); } void reload(int64_t generation) override { _generation = generation; } diff --git a/config/src/vespa/config/set/configsetsource.cpp b/config/src/vespa/config/set/configsetsource.cpp index e6ab890a9df..36c7d70e0d3 100644 --- a/config/src/vespa/config/set/configsetsource.cpp +++ b/config/src/vespa/config/set/configsetsource.cpp @@ -30,16 +30,16 @@ ConfigSetSource::getConfig() AsciiConfigWriter writer(ss); writer.write(*instance); std::vector<vespalib::string> lines(ss.getlines()); - std::string currentMd5(calculateContentMd5(lines)); + std::string currentXxhash64(calculateContentXxhash64(lines)); - if (isGenerationNewer(_generation, _lastState.generation) && currentMd5.compare(_lastState.md5) != 0) { + if (isGenerationNewer(_generation, _lastState.generation) && currentXxhash64.compare(_lastState.xxhash64) != 0) { LOG(debug, "New generation, updating"); - _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, currentMd5), true, _generation))); - _lastState.md5 = currentMd5; + _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, currentXxhash64), true, _generation))); + _lastState.xxhash64 = currentXxhash64; _lastState.generation = _generation; } else { LOG(debug, "Sending timestamp update"); - _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, currentMd5), false, _generation))); + _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, currentXxhash64), false, _generation))); _lastState.generation = _generation; } } diff --git a/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java index c2539b53b28..cc234ea51d7 100644 --- a/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java +++ b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java @@ -98,8 +98,12 @@ public class CppClassBuilder implements ClassBuilder { String newHeader = headerWriter.toString(); String newBody = bodyWriter.toString(); - File headerFile = new File(rootDir, relativePathUnderRoot + "/" + getFileName(root, "h")); - File bodyFile = new File(rootDir, relativePathUnderRoot + "/" + getFileName(root, "cpp")); + String prefix = ""; + if (relativePathUnderRoot != null) { + prefix = relativePathUnderRoot + "/"; + } + File headerFile = new File(rootDir, prefix + getFileName(root, "h")); + File bodyFile = new File(rootDir, prefix + getFileName(root, "cpp")); writeFile(headerFile, newHeader); writeFile(bodyFile, newBody); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java b/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java index ef7d8756228..7a3fa9fe46c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java @@ -49,4 +49,10 @@ public class GetConfigContext { public String logPre() { return TenantRepository.logPre(app); } + + @Override + public String toString() { + return "get config context for application " + app + ", having handler " + requestHandler; + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index dd514e0d843..93148c52c3a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -205,9 +205,8 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica @Override public ConfigResponse resolveConfig(ApplicationId appId, GetConfigRequest req, Optional<Version> vespaVersion) { Application application = getApplication(appId, vespaVersion); - if (log.isLoggable(Level.FINE)) { - log.log(Level.FINE, TenantRepository.logPre(appId) + "Resolving for tenant '" + tenant + "' with handler for application '" + application + "'"); - } + log.log(Level.FINE, () -> TenantRepository.logPre(appId) + "Resolving for tenant '" + tenant + + "' with handler for application '" + application + "'"); return application.resolveConfig(req, responseFactory); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 89987891c61..8dedb5a29c9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -187,8 +187,9 @@ public class ModelContextImpl implements ModelContext { private final double resourceLimitMemory; private final double minNodeRatioPerGroup; private final int metricsproxyNumThreads; - private final boolean enforceRankProfileInheritance; private final boolean newLocationBrokerLogic; + private final boolean containerDumpHeapOnShutdownTimeout; + private final double containerShutdownTimeout; public FeatureFlags(FlagSource source, ApplicationId appId) { this.defaultTermwiseLimit = flagValue(source, appId, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -214,8 +215,9 @@ public class ModelContextImpl implements ModelContext { this.resourceLimitMemory = flagValue(source, appId, PermanentFlags.RESOURCE_LIMIT_MEMORY); this.minNodeRatioPerGroup = flagValue(source, appId, Flags.MIN_NODE_RATIO_PER_GROUP); this.metricsproxyNumThreads = flagValue(source, appId, Flags.METRICSPROXY_NUM_THREADS); - this.enforceRankProfileInheritance = flagValue(source, appId, Flags.ENFORCE_RANK_PROFILE_INHERITANCE); this.newLocationBrokerLogic = flagValue(source, appId, Flags.NEW_LOCATION_BROKER_LOGIC); + this.containerDumpHeapOnShutdownTimeout = flagValue(source, appId, Flags.CONTAINER_DUMP_HEAP_ON_SHUTDOWN_TIMEOUT); + this.containerShutdownTimeout = flagValue(source, appId,Flags.CONTAINER_SHUTDOWN_TIMEOUT); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @@ -245,8 +247,8 @@ public class ModelContextImpl implements ModelContext { @Override public double minNodeRatioPerGroup() { return minNodeRatioPerGroup; } @Override public int metricsproxyNumThreads() { return metricsproxyNumThreads; } @Override public boolean newLocationBrokerLogic() { return newLocationBrokerLogic; } - - @Override public boolean enforceRankProfileInheritance() { return enforceRankProfileInheritance; } + @Override public double containerShutdownTimeout() { return containerShutdownTimeout; } + @Override public boolean containerDumpHeapOnShutdownTimeout() { return containerDumpHeapOnShutdownTimeout; } private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java index d0c08d2747d..fd27dedf041 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java @@ -10,7 +10,8 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; -import java.util.*; +import java.util.List; + /** * A wrapper for {@link Provisioner} to avoid having to expose multitenant diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java index bad03862133..412eafd9882 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java @@ -27,8 +27,8 @@ import java.util.logging.Logger; import static com.yahoo.vespa.config.protocol.SlimeConfigResponse.fromConfigPayload; /** -* @author hmusum -*/ + * @author hmusum + */ class GetConfigProcessor implements Runnable { private static final Logger log = Logger.getLogger(GetConfigProcessor.class.getName()); @@ -69,7 +69,7 @@ class GetConfigProcessor implements Runnable { // TODO: Increment statistics (Metrics) failed counters when requests fail public Pair<GetConfigContext, Long> getConfig(JRTServerConfigRequest request) { - //Request has already been detached + // Request has already been detached if ( ! request.validateParameters()) { // Error code is set in verifyParameters if parameters are not OK. log.log(Level.WARNING, "Parameters for request " + request + " did not validate: " + request.errorCode() + " : " + request.errorMessage()); @@ -140,6 +140,7 @@ class GetConfigProcessor implements Runnable { } return null; } + @Override public void run() { rpcServer.hostLivenessTracker().receivedRequestFrom(request.getClientHostName()); diff --git a/configserver/src/main/sh/start-configserver b/configserver/src/main/sh/start-configserver index f9eb5e3a0a5..965dc8756be 100755 --- a/configserver/src/main/sh/start-configserver +++ b/configserver/src/main/sh/start-configserver @@ -188,7 +188,6 @@ vespa-run-as-vespa-user vespa-runserver -s configserver -r 30 -p $pidfile -- \ -Djdisc.config.file=$cfpfile \ -Djdisc.export.packages= \ -Djdisc.cache.path=$bundlecachedir \ - -Djdisc.debug.resources=false \ -Djdisc.bundle.path=${VESPA_HOME}/lib/jars \ -Djdisc.logger.enabled=true \ -Djdisc.logger.level=ALL \ diff --git a/container-core/src/main/resources/configdefinitions/container.qr.def b/container-core/src/main/resources/configdefinitions/container.qr.def index 9d9b84eb428..08a598bf4bf 100644 --- a/container-core/src/main/resources/configdefinitions/container.qr.def +++ b/container-core/src/main/resources/configdefinitions/container.qr.def @@ -30,3 +30,9 @@ nodeIndex int default=0 ## Force restart of container on deploy, and defer any changes until restart restartOnDeploy bool default=false restart + +## Force heapdump if process is not able to stop within shutdown.timeout +shutdown.dumpHeapOnTimeout bool default=false + +## Timeout for clean shutdown +shutdown.timeout double default=50.0 diff --git a/container-core/src/main/sh/find-pid b/container-core/src/main/sh/find-pid index e1c2195c805..56c387387d4 100755 --- a/container-core/src/main/sh/find-pid +++ b/container-core/src/main/sh/find-pid @@ -77,20 +77,26 @@ findhost set -euo pipefail if (( $# != 1 )); then - echo "Usage: $0 <service-config-id>" >&2 + echo "Usage: $0 <service-name-or-config-id>" >&2 exit 1 fi -readonly config_id=$1 -status=$(vespa-sentinel-cmd list 2>/dev/null | grep "id=\"${config_id}\"") -if [ -z "${status}" ]; then - echo "No service named '${config_id}'" >&2 - exit 1 -fi -readonly pid=$(echo ${status} | cut -d " " -f 4 | cut -d "=" -f 2) -if ! [[ "${pid}" =~ ^[0-9]+$ ]]; then - echo "Could not find valid pid for '${config_id}' (pid='${pid}')" >&2 - exit 1 +readonly service=$1 +readonly pid_file="$VESPA_HOME/var/run/$service.pid" +if [ -f "$pid_file" ]; then + parent_pid=$(cat "$pid_file") + pid=$(pgrep --parent $parent_pid) +else + status=$(vespa-sentinel-cmd list 2>/dev/null | grep --perl-regexp "($service state=|id=\"$service\")") + if [ -z "${status}" ]; then + echo "No service with name or config id '${service}'" >&2 + exit 1 + fi + pid=$(echo ${status} | cut -d " " -f 4 | cut -d "=" -f 2) + if ! [[ "${pid}" =~ ^[0-9]+$ ]]; then + echo "Could not find valid pid for '${service}' (pid='${pid}')" >&2 + exit 1 + fi fi if [ -z "$(ps -p ${pid} -o pid=)" ]; then echo "Could not find process for '${pid}'" >&2 diff --git a/container-disc/pom.xml b/container-disc/pom.xml index 9b6ccd93a41..a362fb11be6 100644 --- a/container-disc/pom.xml +++ b/container-disc/pom.xml @@ -174,14 +174,17 @@ docprocs-jar-with-dependencies.jar, hosted-zone-api-jar-with-dependencies.jar, jdisc-security-filters-jar-with-dependencies.jar, + linguistics-components-jar-with-dependencies.jar, vespaclient-container-plugin-jar-with-dependencies.jar, vespa-athenz-jar-with-dependencies.jar, + container-apache-http-client-bundle-jar-with-dependencies.jar, <!-- Apache http client repackaged as bundle --> + + <!-- Vespa security utils with necessary 3rd party bundles --> security-utils-jar-with-dependencies.jar, - zkfacade-jar-with-dependencies.jar, - zookeeper-server-jar-with-dependencies.jar, - <!-- Apache http client repackaged as bundle --> - container-apache-http-client-bundle-jar-with-dependencies.jar, - <!-- Jetty --> + bcpkix-jdk15on-${bouncycastle.version}.jar, + bcprov-jdk15on-${bouncycastle.version}.jar, + + <!-- Jetty --> alpn-api-${jetty-alpn.version}.jar, http2-server-${jetty.version}.jar, http2-common-${jetty.version}.jar, @@ -198,6 +201,8 @@ jetty-servlets-${jetty.version}.jar, jetty-util-${jetty.version}.jar, jetty-util-ajax-${jetty.version}.jar, + javax.servlet-api-3.1.0.jar, + <!-- Spifly (required for OSGi service loader used by Jetty) --> org.apache.aries.spifly.dynamic.bundle-${spifly.version}.jar, asm-${asm.version}.jar, @@ -206,10 +211,7 @@ asm-tree-${asm.version}.jar, asm-util-${asm.version}.jar, <!-- Spifly end --> - <!-- Misc 3rd party bundles --> - bcpkix-jdk15on-${bouncycastle.version}.jar, - bcprov-jdk15on-${bouncycastle.version}.jar, - javax.servlet-api-3.1.0.jar, + <!-- Jersey 2 + Jackson 2 --> aopalliance-repackaged-${hk2.version}.jar, hk2-api-${hk2.version}.jar, diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index 853224a5b91..a84d2521b8b 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -1,10 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc; +import com.google.common.util.concurrent.AtomicDouble; import com.google.inject.AbstractModule; import com.google.inject.Inject; import com.google.inject.Injector; import com.yahoo.cloud.config.SlobroksConfig; +import com.yahoo.component.Vtag; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.ConfigInstance; @@ -43,7 +45,6 @@ import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.yolean.Exceptions; -import java.io.IOException; import java.util.Collections; import java.util.HashSet; import java.util.IdentityHashMap; @@ -54,6 +55,7 @@ import java.util.Set; import java.util.WeakHashMap; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; import java.util.logging.Logger; @@ -74,6 +76,8 @@ public final class ConfiguredApplication implements Application { private final String configId; private final OsgiFramework osgiFramework; private final com.yahoo.jdisc.Timer timerSingleton; + private final AtomicBoolean dumpHeapOnShutdownTimeout = new AtomicBoolean(false); + private final AtomicDouble shudownTimeoutS = new AtomicDouble(50.0); // Subscriber that is used when this is not a standalone-container. Subscribes // to config to make sure that container will be registered in slobrok (by {@link com.yahoo.jrt.slobrok.api.Register}) // if slobrok config changes (typically slobroks moving to other nodes) @@ -99,7 +103,7 @@ public final class ConfiguredApplication implements Application { static { LogSetup.initVespaLogging("Container"); - log.log(Level.INFO, "Starting container"); + log.log(Level.INFO, "Starting jdisc" + (Vtag.currentVersion.isEmpty() ? "" : " at version " + Vtag.currentVersion)); } /** @@ -133,7 +137,7 @@ public final class ConfiguredApplication implements Application { @Override public void start() { qrConfig = getConfig(QrConfig.class, true); - + reconfigure(qrConfig); hackToInitializeServer(qrConfig); ContainerBuilder builder = createBuilderWithGuiceBindings(); @@ -222,6 +226,7 @@ public final class ConfiguredApplication implements Application { while (true) { subscriber.waitNextGeneration(false); QrConfig newConfig = QrConfig.class.cast(first(subscriber.config().values())); + reconfigure(qrConfig); if (qrConfig.rpc().port() != newConfig.rpc().port()) { com.yahoo.protect.Process.logAndDie( "Rpc port config has changed from " + @@ -235,6 +240,11 @@ public final class ConfiguredApplication implements Application { } } + void reconfigure(QrConfig qrConfig) { + dumpHeapOnShutdownTimeout.set(qrConfig.shutdown().dumpHeapOnTimeout()); + shudownTimeoutS.set(qrConfig.shutdown().timeout()); + } + private void initializeAndActivateContainer(ContainerBuilder builder) { addHandlerBindings(builder, Container.get().getRequestHandlerRegistry(), configurer.getComponent(ApplicationContext.class).discBindingsConfig); @@ -401,13 +411,11 @@ public final class ConfiguredApplication implements Application { private void startShutdownDeadlineExecutor() { shutdownDeadlineExecutor = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("Shutdown deadline timer")); shutdownDeadlineExecutor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); - long delayMillis = 50 * 1000; + long delayMillis = (long)(shudownTimeoutS.get() * 1000.0); shutdownDeadlineExecutor.schedule(() -> { - String heapDumpName = Defaults.getDefaults().underVespaHome("var/crash/java_pid.") + ProcessHandle.current().pid() + ".hprof"; - try { + if (dumpHeapOnShutdownTimeout.get()) { + String heapDumpName = Defaults.getDefaults().underVespaHome("var/crash/java_pid.") + ProcessHandle.current().pid() + ".hprof"; com.yahoo.protect.Process.dumpHeap(heapDumpName, true); - } catch (IOException e) { - log.log(Level.WARNING, "Failed writing heap dump:", e); } com.yahoo.protect.Process.logAndDie( "Timed out waiting for application shutdown. Please check that all your request handlers " + diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh index f09a17de5e5..2d9bbcf21fc 100755 --- a/container-disc/src/main/sh/vespa-start-container-daemon.sh +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -212,7 +212,6 @@ exec $numactlcmd $envcmd java \ -Djdisc.config.file="$cfpfile" \ -Djdisc.export.packages=${jdisc_export_packages} \ -Djdisc.cache.path="$bundlecachedir" \ - -Djdisc.debug.resources=false \ -Djdisc.bundle.path="${VESPA_HOME}/lib/jars" \ -Djdisc.logger.enabled=false \ -Djdisc.logger.level=ALL \ diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/NetworkMultiplexerHolder.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/NetworkMultiplexerHolder.java index 89ecc931efb..48657de1910 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/NetworkMultiplexerHolder.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/NetworkMultiplexerHolder.java @@ -39,7 +39,7 @@ public class NetworkMultiplexerHolder extends AbstractComponent { public void deconstruct() { synchronized (monitor) { if (net != null) { - net.destroy(); + net.disown(); net = null; } destroyed = true; diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java index 922e4140868..009d1619503 100644 --- a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java @@ -9,6 +9,8 @@ import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.handler.RequestDeniedException; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.service.ClientProvider; + +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; @@ -29,9 +31,10 @@ import java.util.logging.Logger; public final class MbusClient extends AbstractResource implements ClientProvider, ReplyHandler { private static final Logger log = Logger.getLogger(MbusClient.class.getName()); + private static final AtomicInteger threadId = new AtomicInteger(0); private final BlockingQueue<MbusRequest> queue = new LinkedBlockingQueue<>(); private final ClientSession session; - private final Thread thread = new Thread(new SenderTask(), "MbusClient"); + private final Thread thread; private volatile boolean done = false; private final ResourceReference sessionReference; @@ -39,6 +42,7 @@ public final class MbusClient extends AbstractResource implements ClientProvider public MbusClient(ClientSession session) { this.session = session; this.sessionReference = session.refer(); + thread = new Thread(new SenderTask(), "mbus-client-" + threadId.getAndIncrement()); } @Override diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java index 67badddddd2..a165a0b8c64 100644 --- a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java @@ -10,13 +10,19 @@ import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.service.CurrentContainer; import com.yahoo.jdisc.service.ServerProvider; + +import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; -import com.yahoo.messagebus.*; + +import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.MessageHandler; +import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.shared.ServerSession; import java.net.URI; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; /** @@ -24,8 +30,9 @@ import java.util.logging.Logger; */ public final class MbusServer extends AbstractResource implements ServerProvider, MessageHandler { + private enum State {INITIALIZING, RUNNING, STOPPED} private final static Logger log = Logger.getLogger(MbusServer.class.getName()); - private final AtomicBoolean running = new AtomicBoolean(false); + private final AtomicReference<State> runState = new AtomicReference<>(State.INITIALIZING); private final CurrentContainer container; private final ServerSession session; private final URI uri; @@ -44,26 +51,33 @@ public final class MbusServer extends AbstractResource implements ServerProvider public void start() { log.log(Level.FINE, "Starting message bus server."); session.connect(); - running.set(true); + runState.set(State.RUNNING); } @Override public void close() { log.log(Level.FINE, "Closing message bus server."); - running.set(false); + runState.set(State.STOPPED); } @Override protected void destroy() { log.log(Level.FINE, "Destroying message bus server."); - running.set(false); + runState.set(State.STOPPED); sessionReference.close(); } @Override public void handleMessage(Message msg) { - if (!running.get()) { - dispatchErrorReply(msg, ErrorCode.SESSION_BUSY, "Session temporarily closed."); + State state = runState.get(); + if (state == State.INITIALIZING) { + dispatchErrorReply(msg, ErrorCode.SESSION_BUSY, "MBusServer not started."); + return; + } + if (state == State.STOPPED) { + // We might need to detect requests originating from the same JVM, as they nede to fail fast + // As they are holding references to the container preventing proper shutdown. + dispatchErrorReply(msg, ErrorCode.SESSION_BUSY, "MBusServer has been closed."); return; } if (msg.getTrace().shouldTrace(6)) { diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/shared/ClientSession.java b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/ClientSession.java index 0964a254cf2..22322a49a78 100644 --- a/container-messagebus/src/main/java/com/yahoo/messagebus/shared/ClientSession.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/ClientSession.java @@ -10,5 +10,5 @@ import com.yahoo.messagebus.Result; */ public interface ClientSession extends SharedResource { - public Result sendMessage(Message msg); + Result sendMessage(Message msg); } diff --git a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java index bf89f3869ed..0c387456cf3 100644 --- a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java @@ -31,6 +31,7 @@ import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; import static com.yahoo.messagebus.ErrorCode.APP_FATAL_ERROR; +import static com.yahoo.messagebus.ErrorCode.SEND_QUEUE_CLOSED; import static com.yahoo.messagebus.ErrorCode.SESSION_BUSY; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.notNullValue; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java index f02ba85c9bf..42a5e2b42be 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; import java.time.Instant; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -80,6 +81,25 @@ public class ZmsClientMock implements ZmsClient { } @Override + public void createTenantResourceGroup(AthenzDomain tenantDomain, AthenzIdentity provider, String resourceGroup, + Set<RoleAction> roleActions) { + log("createTenantResourceGroup(tenantDomain='%s', resourceGroup='%s')", tenantDomain, resourceGroup); + AthenzDbMock.Domain domain = getDomainOrThrow(tenantDomain, true); + ApplicationId applicationId = new ApplicationId(resourceGroup); + if (!domain.applications.containsKey(applicationId)) { + domain.applications.put(applicationId, new AthenzDbMock.Application()); + } + } + + @Override + public Set<RoleAction> getTenantResourceGroups(AthenzDomain tenantDomain, AthenzIdentity provider, String resourceGroup) { + Set<RoleAction> result = new HashSet<>(); + getDomainOrThrow(tenantDomain, true).applications.get(resourceGroup).acl + .forEach((role, roleMembers) -> result.add(new RoleAction(role.roleName, role.roleName))); + return result; + } + + @Override public void addRoleMember(AthenzRole role, AthenzIdentity member, Optional<String> reason) { if ( ! role.roleName().equals("tenancy.vespa.hosting.admin")) throw new IllegalArgumentException("Mock only supports adding tenant admins, not " + role.roleName()); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/NoopRoleService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/NoopRoleService.java index 970a70c6885..341b521212e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/NoopRoleService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/NoopRoleService.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.aws; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.util.List; import java.util.Optional; @@ -12,7 +13,7 @@ import java.util.Optional; public class NoopRoleService implements RoleService { @Override - public Optional<TenantRoles> createTenantRole(TenantName tenant) { + public Optional<TenantRoles> createTenantRole(Tenant tenant) { return Optional.empty(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/RoleService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/RoleService.java index d27fa0a5bd8..61007b9ff46 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/RoleService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/RoleService.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.aws; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.util.List; import java.util.Optional; @@ -11,7 +12,7 @@ import java.util.Optional; */ public interface RoleService { - Optional<TenantRoles> createTenantRole(TenantName tenant); + Optional<TenantRoles> createTenantRole(Tenant tenant); /** Retrieve the names of the tenant roles (host and container). Does not guarantee these roles exist */ TenantRoles getTenantRole(TenantName tenant); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java index 9bcb80f24ee..bbcfb4e35a5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java @@ -139,6 +139,12 @@ public enum JobType { testCdUsCentral2 ("test-cd-us-central-2", Map.of(cd , ZoneId.from("prod" , "cd-us-central-2")), true), + productionCdUsEast1 ("production-cd-us-east-1", + Map.of(cd , ZoneId.from("prod" , "cd-us-east-1"))), + + testCdUsEast1 ("test-cd-us-east-1", + Map.of(cd , ZoneId.from("prod" , "cd-us-east-1")), true), + productionCdUsWest1 ("production-cd-us-west-1", Map.of(cd , ZoneId.from("prod" , "cd-us-west-1"))), diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java index 7539ef3c63a..f650f71c0ec 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneApi; @@ -89,4 +90,7 @@ public interface ZoneRegistry { /** Returns a URL to the controller's api endpoint */ URI apiUrl(); + /** IAM tenant developer role ARN */ + Optional<String> tenantDeveloperRoleArn(TenantName tenant); + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java index 7fa46031c98..7fa46031c98 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java index 1060b118beb..1060b118beb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/DeletedTenant.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/DeletedTenant.java index cf6d73cb8f8..cf6d73cb8f8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/DeletedTenant.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/DeletedTenant.java diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/LastLoginInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/LastLoginInfo.java index 15f2f97e7d1..15f2f97e7d1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/LastLoginInfo.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/LastLoginInfo.java diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java index 80982d70107..80982d70107 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfo.java index a20477d7aab..81c08e1083b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfo.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfo.java @@ -1,10 +1,6 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.tenant; -import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; - -import java.util.ArrayList; -import java.util.List; import java.util.Objects; /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoAddress.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoAddress.java index a12f351abd6..a12f351abd6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoAddress.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoAddress.java diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoBillingContact.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoBillingContact.java index a00dd626f0a..a00dd626f0a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoBillingContact.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoBillingContact.java diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/package-info.java index 9218bfcd850..9218bfcd850 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/package-info.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/package-info.java diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java index ca173437dd1..a7555307a59 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java @@ -33,8 +33,6 @@ public class CuratorArchiveBucketDb { * Policy size limit is 20kb, about 550 bytes for non-tenant related policies. Each tenant * needs about 500 + len(role_arn) bytes, we limit role_arn to 100 characters, so we can * fit about (20k - 550) / 600 ~ 32 tenants per bucket. - * - * This limit is only enforced for public systems as non-public systems does not use tenant specific policies. */ private final static int TENANTS_PER_BUCKET = 30; @@ -86,7 +84,7 @@ public class CuratorArchiveBucketDb { .orElseGet(() -> { // If not, find an existing bucket with space Optional<ArchiveBucket> unfilledBucket = zoneBuckets.stream() - .filter(bucket -> !system.isPublic() || bucket.tenants().size() < TENANTS_PER_BUCKET) + .filter(bucket -> bucket.tenants().size() < TENANTS_PER_BUCKET) .findAny(); // And place the tenant in that bucket. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index 8cc02b5bf69..cbd64ed8bde 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -164,9 +164,20 @@ public class AthenzFacade implements AccessControl { @Override public void deleteTenant(TenantName tenant, Credentials credentials) { AthenzCredentials athenzCredentials = (AthenzCredentials) credentials; - - log("deleteTenancy(tenantDomain=%s, service=%s)", athenzCredentials.domain(), service); - zmsClient.deleteTenancy(athenzCredentials.domain(), service, athenzCredentials.identityToken(), athenzCredentials.accessToken()); + AthenzDomain tenantDomain = athenzCredentials.domain(); + log("deleteTenancy(tenantDomain=%s, service=%s)", tenantDomain, service); + try { + zmsClient.deleteTenancy(tenantDomain, service, athenzCredentials.identityToken(), athenzCredentials.accessToken()); + } catch (ZmsClientException e) { + if (e.getErrorCode() == 404) { + log.log(Level.WARNING, + "Failed to cleanup tenant " + tenant.value() + " with domain '" + tenantDomain.getName() + + "' in Athenz due to non-existing tenant domain", + e); + } else { + throw e; + } + } } @Override @@ -196,8 +207,19 @@ public class AthenzFacade implements AccessControl { AthenzCredentials athenzCredentials = (AthenzCredentials) credentials; log("deleteProviderResourceGroup(tenantDomain=%s, providerDomain=%s, service=%s, resourceGroup=%s)", athenzCredentials.domain(), service.getDomain().getName(), service.getName(), id.application()); - zmsClient.deleteProviderResourceGroup(athenzCredentials.domain(), service, id.application().value(), - athenzCredentials.identityToken(), athenzCredentials.accessToken()); + try { + zmsClient.deleteProviderResourceGroup(athenzCredentials.domain(), service, id.application().value(), + athenzCredentials.identityToken(), athenzCredentials.accessToken()); + } catch (ZmsClientException e) { + if (e.getErrorCode() == 404) { + log.log(Level.WARNING, + "Failed to cleanup application '" + id.serialized() + + "' in Athenz due to non-existing tenant domain or resource group", + e); + } else { + throw e; + } + } } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 652f8630cb6..88d440b52b9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -518,6 +518,8 @@ public class JobController { type, new Versions(platform.orElse(applicationPackage.deploymentSpec().majorVersion() .flatMap(controller.applications()::lastCompatibleVersion) + .or(() -> lastRun.map(run -> run.versions().targetPlatform()) + .filter(controller.readVersionStatus()::isActive)) .orElseGet(controller::readSystemVersion)), version, lastRun.map(run -> run.versions().targetPlatform()), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java index b096a853541..6c4f5ffff9d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java @@ -2,16 +2,25 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.google.common.collect.Maps; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.archive.CuratorArchiveBucketDb; +import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.time.Duration; +import java.util.List; import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; /** @@ -27,6 +36,8 @@ public class ArchiveAccessMaintainer extends ControllerMaintainer { private final ArchiveService archiveService; private final ZoneRegistry zoneRegistry; private final Metric metric; + private final BooleanFlag archiveEnabled; + private final BooleanFlag developerRoleEnabled; public ArchiveAccessMaintainer(Controller controller, Metric metric, Duration interval) { super(controller, interval); @@ -34,6 +45,8 @@ public class ArchiveAccessMaintainer extends ControllerMaintainer { this.archiveService = controller.serviceRegistry().archiveService(); this.zoneRegistry = controller().zoneRegistry(); this.metric = metric; + this.archiveEnabled = Flags.ENABLE_ONPREM_TENANT_S3_ARCHIVE.bindTo(controller().flagSource()); + this.developerRoleEnabled = Flags.ENABLE_TENANT_DEVELOPER_ROLE.bindTo(controller().flagSource()); } @Override @@ -43,22 +56,45 @@ public class ArchiveAccessMaintainer extends ControllerMaintainer { metric.set(bucketCountMetricName, archiveBucketDb.buckets(zoneId).size(), metric.createContext(Map.of("zone", zoneId.value())))); - var tenantArchiveAccessRoles = controller().tenants().asList().stream() - .filter(t -> t instanceof CloudTenant) - .map(t -> (CloudTenant) t) - .filter(t -> t.archiveAccessRole().isPresent()) - .collect(Collectors.toUnmodifiableMap( - Tenant::name, cloudTenant -> cloudTenant.archiveAccessRole().orElseThrow())); - zoneRegistry.zones().controllerUpgraded().ids().forEach(zoneId -> - archiveBucketDb.buckets(zoneId).forEach(archiveBucket -> - archiveService.updateBucketAndKeyPolicy(zoneId, archiveBucket, - Maps.filterEntries(tenantArchiveAccessRoles, - entry -> archiveBucket.tenants().contains(entry.getKey()))) - ) + zoneRegistry.zones().controllerUpgraded().zones().forEach(z -> { + ZoneId zoneId = z.getId(); + var tenantArchiveAccessRoles = tenantArchiveAccessRoles(z); + archiveBucketDb.buckets(zoneId).forEach(archiveBucket -> + archiveService.updateBucketAndKeyPolicy(zoneId, archiveBucket, + Maps.filterEntries(tenantArchiveAccessRoles, + entry -> archiveBucket.tenants().contains(entry.getKey()))) + ); + } ); return 1.0; } + private Map<TenantName, String> tenantArchiveAccessRoles(ZoneApi zone) { + List<Tenant> tenants = controller().tenants().asList(); + if (zoneRegistry.system().isPublic()) { + return tenants.stream() + .filter(t -> t instanceof CloudTenant) + .map(t -> (CloudTenant) t) + .filter(t -> t.archiveAccessRole().isPresent()) + .collect(Collectors.toUnmodifiableMap( + Tenant::name, cloudTenant -> cloudTenant.archiveAccessRole().orElseThrow())); + } else { + return tenants.stream() + .filter(t -> t instanceof AthenzTenant + && enabled(archiveEnabled, t, zone) && enabled(developerRoleEnabled, t, zone)) + .map(Tenant::name) + .collect(Collectors.toUnmodifiableMap( + Function.identity(), t -> zoneRegistry.tenantDeveloperRoleArn(t).orElseThrow())); + + } + } + + private boolean enabled(BooleanFlag flag, Tenant tenant, ZoneApi zone) { + return flag.with(FetchVector.Dimension.TENANT_ID, tenant.name().value()) + .with(FetchVector.Dimension.ZONE_ID, zone.getId().value()) + .value(); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java index 97e9a233f9f..0bf47f85420 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java @@ -46,7 +46,6 @@ public class DeploymentUpgrader extends ControllerMaintainer { Run last = controller().jobController().last(job).get(); Versions target = new Versions(systemVersion, last.versions().targetApplication(), Optional.of(last.versions().targetPlatform()), Optional.of(last.versions().targetApplication())); if ( ! deployment.version().isBefore(target.targetPlatform())) continue; - if ( controller().clock().instant().isBefore(last.start().plus(Duration.ofDays(1)))) continue; if ( ! isLikelyNightFor(job)) continue; log.log(Level.FINE, "Upgrading deployment of " + instance.id() + " in " + deployment.zone()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index cd7ce8c3fa6..98fd0342ecd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -24,8 +24,10 @@ import java.time.temporal.ChronoUnit; import java.util.HashSet; import java.util.Optional; import java.util.OptionalInt; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; /** * Updates refreshed endpoint certificates and triggers redeployment, and deletes unused certificates. @@ -60,6 +62,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { deployRefreshedCertificates(); updateRefreshedCertificates(); deleteUnusedCertificates(); + reportUnmanagedCertificates(); } catch (Exception e) { log.log(LogLevel.ERROR, "Exception caught while maintaining endpoint certificates", e); return 0.0; @@ -134,6 +137,16 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { }); } + private void reportUnmanagedCertificates() { + Set<String> managedRequestIds = curator.readAllEndpointCertificateMetadata().values().stream().map(EndpointCertificateMetadata::requestId).collect(Collectors.toSet()); + + for (EndpointCertificateMetadata cameoCertificateMetadata : endpointCertificateProvider.listCertificates()) { + if (!managedRequestIds.contains(cameoCertificateMetadata.requestId())) { + log.info("Certificate metadata exists with provider but is not managed by controller: " + cameoCertificateMetadata.requestId() + ", " + cameoCertificateMetadata.issuer() + ", " + cameoCertificateMetadata.requestedDnsSans()); + } + } + } + private Lock lock(ApplicationId applicationId) { return curator.lock(TenantAndApplicationId.from(applicationId)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java index ce6f9c802d6..d2b43dc63d9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java @@ -27,9 +27,7 @@ public class TenantRoleMaintainer extends ControllerMaintainer { var tenants = controller().tenants().asList(); // Create separate athenz service for all tenants - tenants.stream() - .map(Tenant::name) - .forEach(roleService::createTenantRole); + tenants.forEach(roleService::createTenantRole); // Until we have moved to separate athenz service per tenant, make sure we update the shared policy // to allow ssh logins for hosts in prod/perf with a separate tenant iam role. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 1a2acb82348..98ac789de04 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -1967,6 +1967,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { .flatMap(options -> optional("vespaVersion", options)) .map(Version::fromString); + ensureApplicationExists(TenantAndApplicationId.from(id), request); + controller.jobController().deploy(id, type, version, applicationPackage); RunId runId = controller.jobController().last(id, type).get().id(); Slime slime = new Slime(); @@ -2617,6 +2619,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { applicationPackage, Optional.of(requireUserPrincipal(request))); + ensureApplicationExists(TenantAndApplicationId.from(tenant, application), request); + return JobControllerApiHandlerHelper.submitResponse(controller.jobController(), tenant, application, @@ -2718,5 +2722,13 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { .anyMatch(definition -> definition == RoleDefinition.hostedOperator); } + private void ensureApplicationExists(TenantAndApplicationId id, HttpRequest request) { + if (controller.applications().getApplication(id).isEmpty()) { + log.fine("Application does not exist in public, creating: " + id); + var credentials = accessControlRequests.credentials(id.tenant(), null /* not used on public */ , request.getJDiscRequest()); + controller.applications().createApplication(id, credentials); + } + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java index 6619b2ff5c6..61bf8feae35 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java @@ -14,8 +14,6 @@ import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; -import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.integration.horizon.HorizonClient; import com.yahoo.vespa.hosted.controller.api.integration.horizon.HorizonResponse; import com.yahoo.vespa.hosted.controller.api.role.Role; @@ -28,7 +26,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.EnumSet; -import java.util.HashSet; import java.util.Optional; import java.util.Set; import java.util.logging.Level; @@ -41,7 +38,6 @@ import java.util.stream.Collectors; */ public class HorizonApiHandler extends LoggingRequestHandler { - private final BillingController billingController; private final SystemName systemName; private final HorizonClient client; private final BooleanFlag enabledHorizonDashboard; @@ -52,7 +48,6 @@ public class HorizonApiHandler extends LoggingRequestHandler { @Inject public HorizonApiHandler(LoggingRequestHandler.Context parentCtx, Controller controller, FlagSource flagSource) { super(parentCtx); - this.billingController = controller.serviceRegistry().billingController(); this.systemName = controller.system(); this.client = controller.serviceRegistry().horizonClient(); this.enabledHorizonDashboard = Flags.ENABLED_HORIZON_DASHBOARD.bindTo(flagSource); @@ -123,13 +118,11 @@ public class HorizonApiHandler extends LoggingRequestHandler { } private Set<TenantName> getAuthorizedTenants(Set<Role> roles) { - var horizonEnabled = roles.stream() + return roles.stream() .filter(TenantRole.class::isInstance) .map(role -> ((TenantRole) role).tenant()) .filter(tenant -> enabledHorizonDashboard.with(FetchVector.Dimension.TENANT_ID, tenant.value()).value()) - .collect(Collectors.toList()); - - return new HashSet<>(billingController.tenantsWithPlan(horizonEnabled, PlanId.from("pay-as-you-go"))); + .collect(Collectors.toSet()); } private static class JsonInputStreamResponse extends HttpResponse { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index 044b7b76d1e..7e88f127026 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -19,10 +19,8 @@ import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeStream; import com.yahoo.slime.SlimeUtils; import com.yahoo.text.Text; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.configserver.flags.FlagsDb; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.IntFlag; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.Controller; @@ -69,18 +67,16 @@ public class UserApiHandler extends LoggingRequestHandler { private final UserManagement users; private final Controller controller; - private final BooleanFlag enable_public_signup_flow; + private final FlagsDb flagsDb; private final IntFlag maxTrialTenants; - private final BooleanFlag enabledHorizonDashboard; @Inject - public UserApiHandler(Context parentCtx, UserManagement users, Controller controller, FlagSource flagSource) { + public UserApiHandler(Context parentCtx, UserManagement users, Controller controller, FlagSource flagSource, FlagsDb flagsDb) { super(parentCtx); this.users = users; this.controller = controller; - this.enable_public_signup_flow = PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.bindTo(flagSource); + this.flagsDb = flagsDb; this.maxTrialTenants = PermanentFlags.MAX_TRIAL_TENANTS.bindTo(flagSource); - this.enabledHorizonDashboard = Flags.ENABLED_HORIZON_DASHBOARD.bindTo(flagSource); } @Override @@ -170,8 +166,6 @@ public class UserApiHandler extends LoggingRequestHandler { root.setBool("isPublic", controller.system().isPublic()); root.setBool("isCd", controller.system().isCd()); - root.setBool(enable_public_signup_flow.id().toString(), - enable_public_signup_flow.with(FetchVector.Dimension.CONSOLE_USER_EMAIL, user.email()).value()); root.setBool("hasTrialCapacity", hasTrialCapacity()); toSlime(root.setObject("user"), user); @@ -186,10 +180,6 @@ public class UserApiHandler extends LoggingRequestHandler { Cursor tenantRolesObject = tenantObject.setArray("roles"); tenantRolesByTenantName.getOrDefault(tenant, List.of()) .forEach(role -> tenantRolesObject.addString(role.definition().name())); - if (controller.system().isPublic()) { - tenantObject.setBool(enabledHorizonDashboard.id().toString(), - enabledHorizonDashboard.with(FetchVector.Dimension.TENANT_ID, tenant.value()).value()); - } }); if (!operatorRoles.isEmpty()) { @@ -197,6 +187,8 @@ public class UserApiHandler extends LoggingRequestHandler { operatorRoles.forEach(role -> operator.addString(role.definition().name())); } + UserFlagsSerializer.toSlime(root, flagsDb.getAllFlagData(), tenantRolesByTenantName.keySet(), !operatorRoles.isEmpty(), user.email()); + return new SlimeJsonResponse(slime); } @@ -249,7 +241,7 @@ public class UserApiHandler extends LoggingRequestHandler { }); } - private void toSlime(Cursor userObject, User user) { + private static void toSlime(Cursor userObject, User user) { if (user.name() != null) userObject.setString("name", user.name()); userObject.setString("email", user.email()); if (user.nickname() != null) userObject.setString("nickname", user.nickname()); @@ -376,7 +368,7 @@ public class UserApiHandler extends LoggingRequestHandler { return Exceptions.uncheck(() -> SlimeUtils.jsonToSlime(IOUtils.readBytes(request.getData(), 1 << 10)).get()); } - private <Type> Type require(String name, Function<Inspector, Type> mapper, Inspector object) { + private static <Type> Type require(String name, Function<Inspector, Type> mapper, Inspector object) { if ( ! object.field(name).valid()) throw new IllegalArgumentException("Missing field '" + name + "'."); return mapper.apply(object.field(name)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializer.java new file mode 100644 index 00000000000..44d537883f9 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializer.java @@ -0,0 +1,86 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.user; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.lang.MutableBoolean; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagDefinition; +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.RawFlag; +import com.yahoo.vespa.flags.UnboundFlag; +import com.yahoo.vespa.flags.json.Condition; +import com.yahoo.vespa.flags.json.FlagData; +import com.yahoo.vespa.flags.json.Rule; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * @author freva + */ +public class UserFlagsSerializer { + static void toSlime(Cursor cursor, Map<FlagId, FlagData> rawFlagData, + Set<TenantName> authorizedForTenantNames, boolean isOperator, String userEmail) { + FetchVector resolveVector = FetchVector.fromMap(Map.of(FetchVector.Dimension.CONSOLE_USER_EMAIL, userEmail)); + List<FlagData> filteredFlagData = Flags.getAllFlags().stream() + // Only include flags that have CONSOLE_USER_EMAIL dimension, this should be replaced with more explicit + // 'target' annotation if/when that is added to flag definition + .filter(fd -> fd.getDimensions().contains(FetchVector.Dimension.CONSOLE_USER_EMAIL)) + .map(FlagDefinition::getUnboundFlag) + .map(flag -> filteredFlagData(flag, Optional.ofNullable(rawFlagData.get(flag.id())), authorizedForTenantNames, isOperator, resolveVector)) + .collect(Collectors.toUnmodifiableList()); + + byte[] bytes = FlagData.serializeListToUtf8Json(filteredFlagData); + SlimeUtils.copyObject(SlimeUtils.jsonToSlime(bytes).get(), cursor); + } + + private static <T> FlagData filteredFlagData(UnboundFlag<T, ?, ?> definition, Optional<FlagData> original, + Set<TenantName> authorizedForTenantNames, boolean isOperator, FetchVector resolveVector) { + MutableBoolean encounteredEmpty = new MutableBoolean(false); + Optional<RawFlag> defaultValue = Optional.of(definition.serializer().serialize(definition.defaultValue())); + // Include the original rules from flag DB and the default value from code if there is no default rule in DB + List<Rule> rules = Stream.concat(original.stream().flatMap(fd -> fd.rules().stream()), Stream.of(new Rule(defaultValue))) + // Exclude rules that do not match the resolveVector + .filter(rule -> rule.partialMatch(resolveVector)) + // Re-create each rule with value explicitly set, either from DB or default from code and + // a filtered set of conditions + .map(rule -> new Rule(rule.getValueToApply().or(() -> defaultValue), + rule.conditions().stream() + .flatMap(condition -> filteredCondition(condition, authorizedForTenantNames, isOperator, resolveVector).stream()) + .collect(Collectors.toUnmodifiableList()))) + // We can stop as soon as we hit the first rule that has no conditions + .takeWhile(rule -> !encounteredEmpty.getAndSet(rule.conditions().isEmpty())) + .collect(Collectors.toUnmodifiableList()); + + return new FlagData(definition.id(), new FetchVector(), rules); + } + + private static Optional<Condition> filteredCondition(Condition condition, Set<TenantName> authorizedForTenantNames, + boolean isOperator, FetchVector resolveVector) { + // If the condition is one of the conditions that we resolve on the server, e.g. email, we do not need to + // propagate it back to the user + if (resolveVector.hasDimension(condition.dimension())) return Optional.empty(); + + // For the other dimensions, filter the values down to an allowed subset + switch (condition.dimension()) { + case TENANT_ID: return valueSubset(condition, tenant -> isOperator || authorizedForTenantNames.contains(TenantName.from(tenant))); + case APPLICATION_ID: return valueSubset(condition, appId -> isOperator || authorizedForTenantNames.stream().anyMatch(tenant -> appId.startsWith(tenant.value() + ":"))); + default: throw new IllegalArgumentException("Dimension " + condition.dimension() + " is not supported for user flags"); + } + } + + private static Optional<Condition> valueSubset(Condition condition, Predicate<String> predicate) { + Condition.CreateParams createParams = condition.toCreateParams(); + return Optional.of(createParams + .withValues(createParams.values().stream().filter(predicate).collect(Collectors.toUnmodifiableList())) + .createAs(condition.type())); + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index fe03b69a3fe..7ce17aff782 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneApi; @@ -216,6 +217,8 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry return URI.create("https://api.tld:4443/"); } + @Override public Optional<String> tenantDeveloperRoleArn(TenantName tenant) { return Optional.empty(); } + @Override public boolean hasZone(ZoneId zoneId) { return zones.stream().anyMatch(zone -> zone.getId().equals(zoneId)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java index ec33c8a7048..1c07a321953 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java @@ -55,19 +55,13 @@ public class DeploymentUpgraderTest { assertEquals(start, tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start()); - // 14 hours pass, but not upgraded before a day has passed since last deployment - tester.clock().advance(Duration.ofHours(14)); + // 11 hours pass, but not upgraded since it's not likely in the middle of the night + tester.clock().advance(Duration.ofHours(11)); upgrader.maintain(); assertEquals(start, tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start()); - // 35 hours pass, but not upgraded since it's not likely in the middle of the night - tester.clock().advance(Duration.ofHours(21)); - upgrader.maintain(); - assertEquals(start, tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); - assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start()); - - // 38 hours pass, and the dev deployment, only, is upgraded + // 14 hours pass, and the dev deployment, only, is upgraded tester.clock().advance(Duration.ofHours(3)); upgrader.maintain(); assertEquals(tester.clock().instant().truncatedTo(MILLIS), tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java index 10f143a8e96..1d844859c37 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java @@ -26,7 +26,6 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Optional; -import java.util.function.Consumer; import java.util.function.Supplier; import java.util.regex.Pattern; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index 5f76a30bf45..8c6d368d93a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.application; +import ai.vespa.hosted.api.MultiPartStreamer; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; @@ -10,10 +11,12 @@ import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; @@ -232,6 +235,42 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { 200); } + @Test + public void create_application_on_deploy() { + var application = ApplicationName.from("unique"); + var applicationPackage = new ApplicationPackageBuilder().build(); + + assertTrue(tester.controller().applications().getApplication(TenantAndApplicationId.from(tenantName, application)).isEmpty()); + + tester.assertResponse( + request("/application/v4/tenant/scoober/application/unique/instance/default/deploy/dev-aws-us-east-1c", POST) + .data(createApplicationDeployData(Optional.of(applicationPackage), Optional.empty(), true)) + .roles(Set.of(Role.developer(tenantName))), + "{\"message\":\"Deployment started in run 1 of dev-aws-us-east-1c for scoober.unique. This may take about 15 minutes the first time.\",\"run\":1}"); + + assertTrue(tester.controller().applications().getApplication(TenantAndApplicationId.from(tenantName, application)).isPresent()); + } + + @Test + public void create_application_on_submit() { + var application = ApplicationName.from("unique"); + var applicationPackage = new ApplicationPackageBuilder() + .trustDefaultCertificate() + .build(); + + assertTrue(tester.controller().applications().getApplication(TenantAndApplicationId.from(tenantName, application)).isEmpty()); + + var data = ApplicationApiTest.createApplicationSubmissionData(applicationPackage, 123); + + tester.assertResponse( + request("/application/v4/tenant/scoober/application/unique/submit", POST) + .data(data) + .roles(Set.of(Role.developer(tenantName))), + "{\"message\":\"Application package version: 1.0.1-commit1, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); + + assertTrue(tester.controller().applications().getApplication(TenantAndApplicationId.from(tenantName, application)).isPresent()); + } + private ApplicationPackageBuilder prodBuilder() { return new ApplicationPackageBuilder() .instances("default") @@ -264,8 +303,31 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { JobType.productionAwsUsEast1c, Optional.empty(), applicationPackage); + } + private MultiPartStreamer createApplicationDeployData(Optional<ApplicationPackage> applicationPackage, + Optional<ApplicationVersion> applicationVersion, boolean deployDirectly) { + MultiPartStreamer streamer = new MultiPartStreamer(); + streamer.addJson("deployOptions", deployOptions(deployDirectly, applicationVersion)); + applicationPackage.ifPresent(ap -> streamer.addBytes("applicationZip", ap.zippedContent())); + return streamer; + } + + private String deployOptions(boolean deployDirectly, Optional<ApplicationVersion> applicationVersion) { + return "{\"vespaVersion\":null," + + "\"ignoreValidationErrors\":false," + + "\"deployDirectly\":" + deployDirectly + + applicationVersion.map(version -> + "," + + "\"buildNumber\":" + version.buildNumber().getAsLong() + "," + + "\"sourceRevision\":{" + + "\"repository\":\"" + version.source().get().repository() + "\"," + + "\"branch\":\"" + version.source().get().branch() + "\"," + + "\"commit\":\"" + version.source().get().commit() + "\"" + + "}" + ).orElse("") + + "}"; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index fb860063696..596a0b186db 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -1,4 +1,4 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.application; import ai.vespa.hosted.api.MultiPartStreamer; @@ -250,7 +250,7 @@ public class ApplicationApiTest extends ControllerContainerTest { var app1 = deploymentTester.newDeploymentContext(id); // POST (deploy) an application to start a manual deployment in prod is not allowed - MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/production-us-east-3/", POST) .data(entity) .userIdentity(USER_ID), @@ -289,7 +289,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST an application package is not generally allowed under user instance tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/otheruser/deploy/dev-us-east-1", POST) .userIdentity(OTHER_USER_ID) - .data(createApplicationDeployData(applicationPackageInstance1, false)), + .data(createApplicationDeployData(applicationPackageInstance1)), accessDenied, 403); @@ -303,7 +303,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST an application package is not allowed under user instance for tenant admins tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/myuser/deploy/dev-us-east-1", POST) .userIdentity(USER_ID) - .data(createApplicationDeployData(applicationPackageInstance1, false)), + .data(createApplicationDeployData(applicationPackageInstance1)), new File("deployment-job-accepted-2.json")); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/myuser/job/dev-us-east-1/diff/1", GET).userIdentity(HOSTED_VESPA_OPERATOR), @@ -1026,38 +1026,24 @@ public class ApplicationApiTest extends ControllerContainerTest { } @Test - public void testDeployDirectly() { + public void testDeployWithApplicationPackage() { // Setup - createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); addUserToHostedOperatorRole(HostedAthenzIdentities.from(HOSTED_VESPA_OPERATOR)); - // Create tenant - tester.assertResponse(request("/application/v4/tenant/tenant1", POST).userIdentity(USER_ID) - .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") - .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), - new File("tenant-without-applications.json")); - - // Create application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", POST) - .userIdentity(USER_ID) - .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), - new File("instance-reference.json")); - - // Add build service to operator role - addUserToHostedOperatorRole(HostedAthenzIdentities.from(SCREWDRIVER_ID)); - // POST (deploy) a system application with an application package - MultiPartStreamer noAppEntity = createApplicationDeployData(Optional.empty(), true); + MultiPartStreamer noAppEntity = createApplicationDeployData(Optional.empty()); tester.assertResponse(request("/application/v4/tenant/hosted-vespa/application/routing/environment/prod/region/us-central-1/instance/default/deploy", POST) .data(noAppEntity) .userIdentity(HOSTED_VESPA_OPERATOR), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Deployment of system applications during a system upgrade is not allowed\"}", 400); - deploymentTester.controllerTester().upgradeSystem(deploymentTester.controller().readVersionStatus().controllerVersion().get().versionNumber()); + deploymentTester.controllerTester() + .upgradeSystem(deploymentTester.controller().readVersionStatus().controllerVersion().get() + .versionNumber()); tester.assertResponse(request("/application/v4/tenant/hosted-vespa/application/routing/environment/prod/region/us-central-1/instance/default/deploy", POST) - .data(noAppEntity) - .userIdentity(HOSTED_VESPA_OPERATOR), - new File("deploy-result.json")); + .data(noAppEntity) + .userIdentity(HOSTED_VESPA_OPERATOR), + new File("deploy-result.json")); } @Test @@ -1113,7 +1099,7 @@ public class ApplicationApiTest extends ControllerContainerTest { } @Test - public void testErrorResponses() throws Exception { + public void testErrorResponses() { createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); // PUT (update) non-existing tenant returns 403 as tenant access cannot be determined when the tenant does not exist @@ -1223,7 +1209,7 @@ public class ApplicationApiTest extends ControllerContainerTest { 400); // POST (deploy) an application to legacy deploy path - MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/instance1/deploy", POST) .data(entity) .userIdentity(USER_ID), @@ -1330,7 +1316,7 @@ public class ApplicationApiTest extends ControllerContainerTest { 200); // Deploy to an authorized zone by a user tenant is disallowed - MultiPartStreamer entity = createApplicationDeployData(applicationPackageDefault, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackageDefault); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/deploy", POST) .data(entity) .userIdentity(USER_ID), @@ -1437,7 +1423,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .build(); createTenantAndApplication(); - MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackage); // POST (deploy) an application to dev through a deployment job, with user instance and a proper tenant tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/new-user/deploy/dev-us-east-1", POST) .data(entity) @@ -1484,7 +1470,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .build(); // deploy the application to a dev zone. Should fail since the developer is not authorized to launch the service - MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackage); tester.assertResponse(request("/application/v4/tenant/sandbox/application/myapp/instance/default/deploy/dev-us-east-1", POST) .data(entity) .userIdentity(developer), @@ -1646,7 +1632,7 @@ public class ApplicationApiTest extends ControllerContainerTest { } @Test - public void testServiceView() throws Exception { + public void testServiceView() { createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); String serviceApi="/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service"; // Not allowed to request apis not listed in feature flag allowed-service-view-apis. e.g /document/v1 @@ -1671,6 +1657,36 @@ public class ApplicationApiTest extends ControllerContainerTest { 403); } + @Test + public void create_application_on_deploy() { + // Setup + createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); + addUserToHostedOperatorRole(HostedAthenzIdentities.from(HOSTED_VESPA_OPERATOR)); + + // Create tenant + tester.assertResponse(request("/application/v4/tenant/tenant1", POST).userIdentity(USER_ID) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") + .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), + new File("tenant-without-applications.json")); + + // Deploy application + var id = ApplicationId.from("tenant1", "application1", "instance1"); + var appId = TenantAndApplicationId.from(id); + var entity = createApplicationDeployData(applicationPackageInstance1); + + assertTrue(tester.controller().applications().getApplication(appId).isEmpty()); + + // POST (deploy) an application to start a manual deployment to dev + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/dev-us-east-1/", POST) + .data(entity) + .oktaIdentityToken(OKTA_IT) + .oktaAccessToken(OKTA_AT) + .userIdentity(USER_ID), + "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1}"); + + assertTrue(tester.controller().applications().getApplication(appId).isPresent()); + } + private static String serializeInstant(Instant i) { return DateTimeFormatter.ISO_INSTANT.format(i.truncatedTo(ChronoUnit.SECONDS)); } @@ -1683,18 +1699,18 @@ public class ApplicationApiTest extends ControllerContainerTest { .build(); } - private MultiPartStreamer createApplicationDeployData(ApplicationPackage applicationPackage, boolean deployDirectly) { - return createApplicationDeployData(Optional.of(applicationPackage), deployDirectly); + private MultiPartStreamer createApplicationDeployData(ApplicationPackage applicationPackage) { + return createApplicationDeployData(Optional.of(applicationPackage)); } - private MultiPartStreamer createApplicationDeployData(Optional<ApplicationPackage> applicationPackage, boolean deployDirectly) { - return createApplicationDeployData(applicationPackage, Optional.empty(), deployDirectly); + private MultiPartStreamer createApplicationDeployData(Optional<ApplicationPackage> applicationPackage) { + return createApplicationDeployData(applicationPackage, Optional.empty()); } private MultiPartStreamer createApplicationDeployData(Optional<ApplicationPackage> applicationPackage, - Optional<ApplicationVersion> applicationVersion, boolean deployDirectly) { + Optional<ApplicationVersion> applicationVersion) { MultiPartStreamer streamer = new MultiPartStreamer(); - streamer.addJson("deployOptions", deployOptions(deployDirectly, applicationVersion)); + streamer.addJson("deployOptions", deployOptions(applicationVersion)); applicationPackage.ifPresent(ap -> streamer.addBytes("applicationZip", ap.zippedContent())); return streamer; } @@ -1706,10 +1722,9 @@ public class ApplicationApiTest extends ControllerContainerTest { .addBytes(EnvironmentResource.APPLICATION_TEST_ZIP, "content".getBytes()); } - private String deployOptions(boolean deployDirectly, Optional<ApplicationVersion> applicationVersion) { + private String deployOptions(Optional<ApplicationVersion> applicationVersion) { return "{\"vespaVersion\":null," + - "\"ignoreValidationErrors\":false," + - "\"deployDirectly\":" + deployDirectly + + "\"ignoreValidationErrors\":false" + applicationVersion.map(version -> "," + "\"buildNumber\":" + version.buildNumber().getAsLong() + "," + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiTest.java index 8e51f8210c7..b2b5b2286f7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiTest.java @@ -4,7 +4,6 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; -import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerCloudTest; @@ -33,8 +32,6 @@ public class HorizonApiTest extends ControllerContainerCloudTest { ((InMemoryFlagSource) tester.controller().flagSource()) .withBooleanFlag(Flags.ENABLED_HORIZON_DASHBOARD.id(), true); - tester.controller().serviceRegistry().billingController().setPlan(tenantName, PlanId.from("pay-as-you-go"), true); - tester.assertResponse(request("/horizon/v1/config/dashboard/topFolders") .roles(Set.of(Role.reader(tenantName))), "", 200); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java index acd481030e2..c884eae8afc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java @@ -5,6 +5,8 @@ import com.yahoo.application.container.handler.Request; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.utils.AthenzIdentities; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.user.User; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; @@ -25,40 +27,42 @@ public class UserApiOnPremTest extends ControllerContainerTest { @Test public void userMetadataOnPremTest() { - ContainerTester tester = new ContainerTester(container, responseFiles); - ControllerTester controller = new ControllerTester(tester); - User user = new User("dev@domail", "Joe Developer", "dev", null); + try (Flags.Replacer ignored = Flags.clearFlagsForTesting(PermanentFlags.MAX_TRIAL_TENANTS.id(), PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.id())) { + ContainerTester tester = new ContainerTester(container, responseFiles); + ControllerTester controller = new ControllerTester(tester); + User user = new User("dev@domail", "Joe Developer", "dev", null); - controller.createTenant("tenant1", "domain1", 1L); - controller.createApplication("tenant1", "app1", "default"); - controller.createApplication("tenant1", "app2", "default"); - controller.createApplication("tenant1", "app2", "myinstance"); - controller.createApplication("tenant1", "app3"); + controller.createTenant("tenant1", "domain1", 1L); + controller.createApplication("tenant1", "app1", "default"); + controller.createApplication("tenant1", "app2", "default"); + controller.createApplication("tenant1", "app2", "myinstance"); + controller.createApplication("tenant1", "app3"); - controller.createTenant("tenant2", "domain2", 2L); - controller.createApplication("tenant2", "app2", "test"); + controller.createTenant("tenant2", "domain2", 2L); + controller.createApplication("tenant2", "app2", "test"); - controller.createTenant("tenant3", "domain3", 3L); - controller.createApplication("tenant3", "app1"); + controller.createTenant("tenant3", "domain3", 3L); + controller.createApplication("tenant3", "app1"); - controller.createTenant("sandbox", "domain4", 4L); - controller.createApplication("sandbox", "app1", "default"); - controller.createApplication("sandbox", "app2", "default"); - controller.createApplication("sandbox", "app2", "dev"); + controller.createTenant("sandbox", "domain4", 4L); + controller.createApplication("sandbox", "app1", "default"); + controller.createApplication("sandbox", "app2", "default"); + controller.createApplication("sandbox", "app2", "dev"); - AthenzIdentity operator = AthenzIdentities.from("vespa.alice"); - controller.athenzDb().addHostedOperator(operator); - AthenzIdentity tenantAdmin = AthenzIdentities.from("domain1.bob"); - Stream.of("domain1", "domain2", "domain4") - .map(AthenzDomain::new) - .map(controller.athenzDb()::getOrCreateDomain) - .forEach(d -> d.admin(AthenzIdentities.from("domain1.bob"))); + AthenzIdentity operator = AthenzIdentities.from("vespa.alice"); + controller.athenzDb().addHostedOperator(operator); + AthenzIdentity tenantAdmin = AthenzIdentities.from("domain1.bob"); + Stream.of("domain1", "domain2", "domain4") + .map(AthenzDomain::new) + .map(controller.athenzDb()::getOrCreateDomain) + .forEach(d -> d.admin(AthenzIdentities.from("domain1.bob"))); - tester.assertResponse(createUserRequest(user, operator), - new File("user-without-applications.json")); + tester.assertResponse(createUserRequest(user, operator), + new File("user-without-applications.json")); - tester.assertResponse(createUserRequest(user, tenantAdmin), - new File("user-with-applications-athenz.json")); + tester.assertResponse(createUserRequest(user, tenantAdmin), + new File("user-with-applications-athenz.json")); + } } private Request createUserRequest(User user, AthenzIdentity identity) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index 03f1d75a50b..9198369a3ad 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.restapi.user; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.ControllerTester; @@ -205,65 +206,68 @@ public class UserApiTest extends ControllerContainerCloudTest { @Test public void userMetadataTest() { - ContainerTester tester = new ContainerTester(container, responseFiles); - ((InMemoryFlagSource) tester.controller().flagSource()) - .withBooleanFlag(PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.id(), true); - ControllerTester controller = new ControllerTester(tester); - Set<Role> operator = Set.of(Role.hostedOperator(), Role.hostedSupporter(), Role.hostedAccountant()); - User user = new User("dev@domail", "Joe Developer", "dev", null); - - tester.assertResponse(request("/user/v1/user") - .roles(operator) - .user(user), - new File("user-without-applications.json")); - - controller.createTenant("tenant1", Tenant.Type.cloud); - controller.createApplication("tenant1", "app1", "default"); - controller.createApplication("tenant1", "app2", "default"); - controller.createApplication("tenant1", "app2", "myinstance"); - controller.createApplication("tenant1", "app3"); - - controller.createTenant("tenant2", Tenant.Type.cloud); - controller.createApplication("tenant2", "app2", "test"); - - controller.createTenant("tenant3", Tenant.Type.cloud); - controller.createApplication("tenant3", "app1"); - - controller.createTenant("sandbox", Tenant.Type.cloud); - controller.createApplication("sandbox", "app1", "default"); - controller.createApplication("sandbox", "app2", "default"); - controller.createApplication("sandbox", "app2", "dev"); - - // Should still be empty because none of the roles explicitly refer to any of the applications - tester.assertResponse(request("/user/v1/user") - .roles(operator) - .user(user), - new File("user-without-applications.json")); - - // Empty applications because tenant dummy does not exist - tester.assertResponse(request("/user/v1/user") - .roles(Set.of(Role.administrator(TenantName.from("tenant1")), - Role.developer(TenantName.from("tenant2")), - Role.developer(TenantName.from("sandbox")), - Role.reader(TenantName.from("sandbox")))) - .user(user), - new File("user-with-applications-cloud.json")); + try (Flags.Replacer ignored = Flags.clearFlagsForTesting(PermanentFlags.MAX_TRIAL_TENANTS.id(), PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.id())) { + ContainerTester tester = new ContainerTester(container, responseFiles); + ((InMemoryFlagSource) tester.controller().flagSource()) + .withBooleanFlag(PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.id(), true); + ControllerTester controller = new ControllerTester(tester); + Set<Role> operator = Set.of(Role.hostedOperator(), Role.hostedSupporter(), Role.hostedAccountant()); + User user = new User("dev@domail", "Joe Developer", "dev", null); + + tester.assertResponse(request("/user/v1/user") + .roles(operator) + .user(user), + new File("user-without-applications.json")); + + controller.createTenant("tenant1", Tenant.Type.cloud); + controller.createApplication("tenant1", "app1", "default"); + controller.createApplication("tenant1", "app2", "default"); + controller.createApplication("tenant1", "app2", "myinstance"); + controller.createApplication("tenant1", "app3"); + + controller.createTenant("tenant2", Tenant.Type.cloud); + controller.createApplication("tenant2", "app2", "test"); + + controller.createTenant("tenant3", Tenant.Type.cloud); + controller.createApplication("tenant3", "app1"); + + controller.createTenant("sandbox", Tenant.Type.cloud); + controller.createApplication("sandbox", "app1", "default"); + controller.createApplication("sandbox", "app2", "default"); + controller.createApplication("sandbox", "app2", "dev"); + + // Should still be empty because none of the roles explicitly refer to any of the applications + tester.assertResponse(request("/user/v1/user") + .roles(operator) + .user(user), + new File("user-without-applications.json")); + + // Empty applications because tenant dummy does not exist + tester.assertResponse(request("/user/v1/user") + .roles(Set.of(Role.administrator(TenantName.from("tenant1")), + Role.developer(TenantName.from("tenant2")), + Role.developer(TenantName.from("sandbox")), + Role.reader(TenantName.from("sandbox")))) + .user(user), + new File("user-with-applications-cloud.json")); + } } @Test public void maxTrialTenants() { - ContainerTester tester = new ContainerTester(container, responseFiles); - ((InMemoryFlagSource) tester.controller().flagSource()) - .withIntFlag(PermanentFlags.MAX_TRIAL_TENANTS.id(), 1) - .withBooleanFlag(PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.id(), true); - ControllerTester controller = new ControllerTester(tester); - Set<Role> operator = Set.of(Role.hostedOperator(), Role.hostedSupporter(), Role.hostedAccountant()); - User user = new User("dev@domail", "Joe Developer", "dev", null); - - controller.createTenant("tenant1", Tenant.Type.cloud); - - tester.assertResponse( - request("/user/v1/user").user(user), - new File("user-without-trial-capacity-cloud.json")); + try (Flags.Replacer ignored = Flags.clearFlagsForTesting(PermanentFlags.MAX_TRIAL_TENANTS.id(), PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.id())) { + ContainerTester tester = new ContainerTester(container, responseFiles); + ((InMemoryFlagSource) tester.controller().flagSource()) + .withIntFlag(PermanentFlags.MAX_TRIAL_TENANTS.id(), 1) + .withBooleanFlag(PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.id(), true); + ControllerTester controller = new ControllerTester(tester); + User user = new User("dev@domail", "Joe Developer", "dev", null); + + controller.createTenant("tenant1", Tenant.Type.cloud); + + tester.assertResponse( + request("/user/v1/user").user(user), + new File("user-without-trial-capacity-cloud.json")); + } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializerTest.java new file mode 100644 index 00000000000..8625628b74e --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializerTest.java @@ -0,0 +1,133 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.user; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.config.provision.TenantName; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.test.json.JsonTestHelper; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.JsonNodeRawFlag; +import com.yahoo.vespa.flags.json.Condition; +import com.yahoo.vespa.flags.json.FlagData; +import com.yahoo.vespa.flags.json.Rule; +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; +import static com.yahoo.vespa.flags.FetchVector.Dimension.CONSOLE_USER_EMAIL; +import static com.yahoo.vespa.flags.FetchVector.Dimension.TENANT_ID; + +/** + * @author freva + */ +public class UserFlagsSerializerTest { + + @Test + public void user_flag_test() throws IOException { + String email1 = "alice@domain.tld"; + String email2 = "bob@domain.tld"; + + try (Flags.Replacer ignored = Flags.clearFlagsForTesting()) { + Flags.defineStringFlag("string-id", "default value", List.of("owner"), "1970-01-01", "2100-01-01", "desc", "mod", CONSOLE_USER_EMAIL); + Flags.defineIntFlag("int-id", 123, List.of("owner"), "1970-01-01", "2100-01-01", "desc", "mod", CONSOLE_USER_EMAIL, TENANT_ID, APPLICATION_ID); + Flags.defineDoubleFlag("double-id", 3.14d, List.of("owner"), "1970-01-01", "2100-01-01", "desc", "mod"); + Flags.defineListFlag("list-id", List.of("a"), String.class, List.of("owner"), "1970-01-01", "2100-01-01", "desc", "mod", CONSOLE_USER_EMAIL); + Flags.defineJacksonFlag("jackson-id", new ExampleJacksonClass(123, "abc"), ExampleJacksonClass.class, + List.of("owner"), "1970-01-01", "2100-01-01", "desc", "mod", CONSOLE_USER_EMAIL, TENANT_ID); + + Map<FlagId, FlagData> flagData = Stream.of( + flagData("string-id", rule("\"value1\"", condition(CONSOLE_USER_EMAIL, Condition.Type.WHITELIST, email1))), + flagData("int-id", rule("456")), + flagData("list-id", + rule("[\"value1\"]", condition(CONSOLE_USER_EMAIL, Condition.Type.WHITELIST, email1), condition(APPLICATION_ID, Condition.Type.BLACKLIST, "tenant1:video:default", "tenant1:video:default", "tenant2:music:default")), + rule("[\"value2\"]", condition(CONSOLE_USER_EMAIL, Condition.Type.WHITELIST, email2)), + rule("[\"value1\",\"value3\"]", condition(APPLICATION_ID, Condition.Type.BLACKLIST, "tenant1:video:default", "tenant1:video:default", "tenant2:music:default"))), + flagData("jackson-id", rule("{\"integer\":456,\"string\":\"xyz\"}", condition(CONSOLE_USER_EMAIL, Condition.Type.WHITELIST, email1), condition(TENANT_ID, Condition.Type.WHITELIST, "tenant1", "tenant3"))) + ).collect(Collectors.toMap(FlagData::id, fd -> fd)); + + // double-id is not here as it does not have CONSOLE_USER_EMAIL dimension + assertUserFlags("{\"flags\":[" + + "{\"id\":\"int-id\",\"rules\":[{\"value\":456}]}," + // Default from DB + "{\"id\":\"jackson-id\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"tenant\"}],\"value\":{\"integer\":456,\"string\":\"xyz\"}},{\"value\":{\"integer\":123,\"string\":\"abc\"}}]}," + // Resolved for email + // Resolved for email, but conditions are empty since this user is not authorized for any tenants + "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\"}],\"value\":[\"value1\"]},{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\"}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + + "{\"id\":\"string-id\",\"rules\":[{\"value\":\"value1\"}]}]}", // resolved for email + flagData, Set.of(), false, email1); + + // Same as the first one, but user is authorized for tenant1 + assertUserFlags("{\"flags\":[" + + "{\"id\":\"int-id\",\"rules\":[{\"value\":456}]}," + // Default from DB + "{\"id\":\"jackson-id\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"tenant\",\"values\":[\"tenant1\"]}],\"value\":{\"integer\":456,\"string\":\"xyz\"}},{\"value\":{\"integer\":123,\"string\":\"abc\"}}]}," + // Resolved for email + // Resolved for email, but conditions have filtered out tenant2 + "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\"]}],\"value\":[\"value1\"]},{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\"]}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + + "{\"id\":\"string-id\",\"rules\":[{\"value\":\"value1\"}]}]}", // resolved for email + flagData, Set.of("tenant1"), false, email1); + + // As operator no conditions are filtered, but the email precondition is applied + assertUserFlags("{\"flags\":[" + + "{\"id\":\"int-id\",\"rules\":[{\"value\":456}]}," + // Default from DB + "{\"id\":\"jackson-id\",\"rules\":[{\"value\":{\"integer\":123,\"string\":\"abc\"}}]}," + // Default from code, no DB values match + // Includes last value from DB which is not conditioned on email and the default from code + "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\",\"tenant2:music:default\"]}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + + "{\"id\":\"string-id\",\"rules\":[{\"value\":\"default value\"}]}]}", // Default from code + flagData, Set.of(), true, "operator@domain.tld"); + } + } + + private static FlagData flagData(String id, Rule... rules) { + return new FlagData(new FlagId(id), new FetchVector(), rules); + } + + private static Rule rule(String data, Condition... conditions) { + return new Rule(Optional.ofNullable(data).map(JsonNodeRawFlag::fromJson), conditions); + } + + private static Condition condition(FetchVector.Dimension dimension, Condition.Type type, String... values) { + return new Condition.CreateParams(dimension).withValues(values).createAs(type); + } + + private static void assertUserFlags(String expected, Map<FlagId, FlagData> rawFlagData, + Set<String> authorizedForTenantNames, boolean isOperator, String userEmail) throws IOException { + Slime slime = new Slime(); + UserFlagsSerializer.toSlime(slime.setObject(), rawFlagData, authorizedForTenantNames.stream().map(TenantName::from).collect(Collectors.toSet()), isOperator, userEmail); + JsonTestHelper.assertJsonEquals(expected, + new String(SlimeUtils.toJsonBytes(slime), StandardCharsets.UTF_8)); + } + + @JsonIgnoreProperties(ignoreUnknown = true) + private static class ExampleJacksonClass { + @JsonProperty("integer") public final int integer; + @JsonProperty("string") public final String string; + private ExampleJacksonClass(@JsonProperty("integer") int integer, @JsonProperty("string") String string) { + this.integer = integer; + this.string = string; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ExampleJacksonClass that = (ExampleJacksonClass) o; + return integer == that.integer && + Objects.equals(string, that.string); + } + + @Override + public int hashCode() { + return Objects.hash(integer, string); + } + } +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json index 5d3a38334ad..006c3b98a4d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json @@ -1,7 +1,6 @@ { "isPublic": false, "isCd": false, - "enable-public-signup-flow": (ignore), "hasTrialCapacity": (ignore), "user": { "name": "Joe Developer", @@ -31,5 +30,6 @@ "reader" ] } - } + }, + "flags": [{"id":"enable-public-signup-flow","rules":[{"value":false}]}] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json index e883993cb53..4ae55e97baa 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json @@ -1,7 +1,6 @@ { "isPublic": true, "isCd": false, - "enable-public-signup-flow": (ignore), "hasTrialCapacity": true, "user": { "name": "Joe Developer", @@ -14,20 +13,18 @@ "roles": [ "developer", "reader" - ], - "enabled-horizon-dashboard":false + ] }, "tenant1": { "roles": [ "administrator" - ], - "enabled-horizon-dashboard":false + ] }, "tenant2": { "roles": [ "developer" - ], - "enabled-horizon-dashboard":false + ] } - } + }, + "flags": [{"id":"enable-public-signup-flow","rules":[{"value":false}]}] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json index 3bf999b490b..9f9578e6ed8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json @@ -1,7 +1,6 @@ { "isPublic": (ignore), "isCd": (ignore), - "enable-public-signup-flow": (ignore), "hasTrialCapacity": (ignore), "user": { "name": "Joe Developer", @@ -14,5 +13,6 @@ "hostedOperator", "hostedSupporter", "hostedAccountant" - ] + ], + "flags": [{"id":"enable-public-signup-flow","rules":[{"value":false}]}] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-trial-capacity-cloud.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-trial-capacity-cloud.json index 27242424579..2b98a75068a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-trial-capacity-cloud.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-trial-capacity-cloud.json @@ -1,7 +1,6 @@ { "isPublic": true, "isCd": false, - "enable-public-signup-flow": true, "hasTrialCapacity": false, "user": { "name": "Joe Developer", @@ -9,5 +8,6 @@ "nickname": "dev", "verified":false }, - "tenants": {} + "tenants": {}, + "flags": [{"id":"enable-public-signup-flow","rules":[{"value":false}]}] }
\ No newline at end of file diff --git a/dist/vespa.spec b/dist/vespa.spec index e976d710fb5..1fa9fbc9796 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -79,6 +79,7 @@ BuildRequires: cmake3 BuildRequires: llvm7.0-devel BuildRequires: vespa-boost-devel >= 1.76.0-1 BuildRequires: vespa-gtest >= 1.8.1-1 +%define _use_vespa_gtest 1 BuildRequires: vespa-icu-devel >= 65.1.0-1 BuildRequires: vespa-lz4-devel >= 1.9.2-2 BuildRequires: vespa-onnxruntime-devel = 1.7.1 @@ -108,6 +109,7 @@ BuildRequires: vespa-boost-devel >= 1.76.0-1 BuildRequires: vespa-openssl-devel >= 1.1.1l-1 %define _use_vespa_openssl 1 BuildRequires: vespa-gtest >= 1.8.1-1 +%define _use_vespa_gtest 1 BuildRequires: vespa-lz4-devel >= 1.9.2-2 BuildRequires: vespa-onnxruntime-devel = 1.7.1 BuildRequires: vespa-protobuf-devel = 3.17.3 @@ -233,6 +235,7 @@ Requires: llvm7.0 Requires: vespa-telegraf >= 1.1.1-1 Requires: vespa-valgrind >= 3.17.0-1 %endif +Requires: vespa-gtest >= 1.8.1-1 %define _vespa_llvm_version 7 %define _extra_link_directory /usr/lib64/llvm7.0/lib;%{_vespa_deps_prefix}/lib64 %define _extra_include_directory /usr/include/llvm7.0;%{_vespa_deps_prefix}/include @@ -247,10 +250,12 @@ Requires: vespa-valgrind >= 3.17.0-1 %else %define _vespa_llvm_version 10 %endif +Requires: vespa-gtest >= 1.8.1-1 %define _extra_link_directory %{_vespa_deps_prefix}/lib64 %define _extra_include_directory %{_vespa_deps_prefix}/include %endif %if 0%{?fedora} +Requires: gtest %if 0%{?fc32} %define _vespa_llvm_version 10 %endif @@ -285,7 +290,7 @@ Requires: %{name}-tools = %{version}-%{release} # Ugly workaround because vespamalloc/src/vespamalloc/malloc/mmap.cpp uses the private # _dl_sym function. # Exclude automated requires for libraries in /opt/vespa-deps/lib64. -%global __requires_exclude ^lib(c\\.so\\.6\\(GLIBC_PRIVATE\\)|pthread\\.so\\.0\\(GLIBC_PRIVATE\\)|(icui18n|icuuc|lz4|protobuf|zstd|onnxruntime%{?_use_vespa_openssl:|crypto|ssl}%{?_use_vespa_openblas:|openblas}%{?_use_vespa_re2:|re2}%{?_use_vespa_xxhash:|xxhash})\\.so\\.[0-9.]*\\([A-Za-z._0-9]*\\))\\(64bit\\)$ +%global __requires_exclude ^lib(c\\.so\\.6\\(GLIBC_PRIVATE\\)|pthread\\.so\\.0\\(GLIBC_PRIVATE\\)|(icui18n|icuuc|lz4|protobuf|zstd|onnxruntime%{?_use_vespa_openssl:|crypto|ssl}%{?_use_vespa_openblas:|openblas}%{?_use_vespa_re2:|re2}%{?_use_vespa_xxhash:|xxhash}%{?_use_vespa_gtest:|(gtest|gmock)(_main)?})\\.so\\.[0-9.]*\\([A-Za-z._0-9]*\\))\\(64bit\\)$ %description @@ -809,6 +814,7 @@ fi %{_prefix}/lib/jars/jdisc_core-jar-with-dependencies.jar %{_prefix}/lib/jars/jdisc-security-filters-jar-with-dependencies.jar %{_prefix}/lib/jars/jersey-*.jar +%{_prefix}/lib/jars/linguistics-components-jar-with-dependencies.jar %{_prefix}/lib/jars/alpn-*.jar %{_prefix}/lib/jars/http2-*.jar %{_prefix}/lib/jars/jetty-*.jar diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java index 8f3f75af795..fa5f794f652 100644 --- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java +++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java @@ -10,13 +10,13 @@ import com.yahoo.language.process.Encoder; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.indexinglanguage.ScriptParserContext; import com.yahoo.vespa.indexinglanguage.expressions.InputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; import com.yahoo.vespa.indexinglanguage.parser.IndexingInput; import com.yahoo.vespa.indexinglanguage.parser.ParseException; import java.util.*; -import java.util.logging.Level; /** * @author Simon Thoresen Hult @@ -86,11 +86,17 @@ public class ScriptManager { List<StatementExpression> expressions = new ArrayList<>(ilscript.content().size()); Map<String, DocumentScript> fieldScripts = new HashMap<>(ilscript.content().size()); for (String content : ilscript.content()) { - expressions.add(parse(ilscript.doctype(), parserContext, content)); StatementExpression statement = parse(ilscript.doctype(), parserContext, content); + expressions.add(statement); InputExpression.InputFieldNameExtractor inputFieldNameExtractor = new InputExpression.InputFieldNameExtractor(); statement.select(inputFieldNameExtractor, inputFieldNameExtractor); + OutputExpression.OutputFieldNameExtractor outputFieldNameExtractor = new OutputExpression.OutputFieldNameExtractor(); + statement.select(outputFieldNameExtractor, outputFieldNameExtractor); statement.select(fieldPathOptimizer, fieldPathOptimizer); + if ( ! outputFieldNameExtractor.getOutputFieldNames().isEmpty()) { + String outputFieldName = outputFieldNameExtractor.getOutputFieldNames().get(0); + statement.setStatementOutputType(docTypeMgr.getDocumentType(ilscript.doctype()).getField(outputFieldName).getDataType()); + } if (inputFieldNameExtractor.getInputFieldNames().size() == 1) { String fieldName = inputFieldNameExtractor.getInputFieldNames().get(0); ScriptExpression script; diff --git a/docprocs/src/test/java/com/yahoo/docprocs/indexing/ScriptManagerTestCase.java b/docprocs/src/test/java/com/yahoo/docprocs/indexing/ScriptManagerTestCase.java index 607fee4f10d..ec05fcbe422 100644 --- a/docprocs/src/test/java/com/yahoo/docprocs/indexing/ScriptManagerTestCase.java +++ b/docprocs/src/test/java/com/yahoo/docprocs/indexing/ScriptManagerTestCase.java @@ -28,7 +28,7 @@ public class ScriptManagerTestCase { IlscriptsConfig.Builder config = new IlscriptsConfig.Builder(); config.ilscript(new IlscriptsConfig.Ilscript.Builder().doctype("newssummary") - .content("index")); + .content("input title | index title")); ScriptManager scriptMgr = new ScriptManager(typeMgr, new IlscriptsConfig(config), null, Encoder.throwsOnUse); assertNotNull(scriptMgr.getScript(typeMgr.getDocumentType("newsarticle"))); assertNull(scriptMgr.getScript(new DocumentType("unknown"))); @@ -43,7 +43,7 @@ public class ScriptManagerTestCase { IlscriptsConfig.Builder config = new IlscriptsConfig.Builder(); config.ilscript(new IlscriptsConfig.Ilscript.Builder().doctype("newsarticle") - .content("index")); + .content("input title | index title")); ScriptManager scriptMgr = new ScriptManager(typeMgr, new IlscriptsConfig(config), null, Encoder.throwsOnUse); assertNotNull(scriptMgr.getScript(typeMgr.getDocumentType("newssummary"))); assertNull(scriptMgr.getScript(new DocumentType("unknown"))); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentMessage.java index f65550bbf0f..e0515a84b5d 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentMessage.java @@ -16,13 +16,6 @@ public class GetDocumentMessage extends DocumentMessage { private String fieldSet = DEFAULT_FIELD_SET; /** - * Constructs a new message for deserialization. - */ - GetDocumentMessage() { - // empty - } - - /** * Constructs a new document get message. * * @param documentId The identifier of the document to get. diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/PutDocumentMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/PutDocumentMessage.java index b2bf26d3b05..58bc50088d7 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/PutDocumentMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/PutDocumentMessage.java @@ -19,13 +19,6 @@ public class PutDocumentMessage extends TestAndSetMessage { private LazyDecoder decoder = null; /** - * Constructs a new message for deserialization. - */ - PutDocumentMessage() { - // empty - } - - /** * Constructs a new message from a byte buffer. * * @param decoder The decoder to use for deserialization. diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 16dd729b1f1..8a7cd4f66bd 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -46,6 +46,7 @@ vespa_define_module( src/tests/gp/ponder_nov2017 src/tests/instruction/add_trivial_dimension_optimizer src/tests/instruction/dense_dot_product_function + src/tests/instruction/dense_hamming_distance src/tests/instruction/dense_inplace_join_function src/tests/instruction/dense_matmul_function src/tests/instruction/dense_multi_matmul_function diff --git a/eval/src/apps/tensor_conformance/generate.cpp b/eval/src/apps/tensor_conformance/generate.cpp index 8a596ad38d4..d0767b45224 100644 --- a/eval/src/apps/tensor_conformance/generate.cpp +++ b/eval/src/apps/tensor_conformance/generate.cpp @@ -14,6 +14,19 @@ using vespalib::make_string_short::fmt; namespace { +struct IgnoreJava : TestBuilder { + TestBuilder &dst; + IgnoreJava(TestBuilder &dst_in) : TestBuilder(dst_in.full), dst(dst_in) {} + void add(const vespalib::string &expression, + const std::map<vespalib::string,TensorSpec> &inputs, + const std::set<vespalib::string> &ignore) override + { + auto my_ignore = ignore; + my_ignore.insert("vespajlib"); + dst.add(expression, inputs, my_ignore); + } +}; + //----------------------------------------------------------------------------- const std::vector<vespalib::string> basic_layouts = { @@ -273,6 +286,9 @@ void generate_join(TestBuilder &dst) { generate_op2_join("min(a,b)", Div16(N()), dst); generate_op2_join("max(a,b)", Div16(N()), dst); generate_op2_join("bit(a,b)", Seq({-128, -43, -1, 0, 85, 127}), Seq({0, 1, 2, 3, 4, 5, 6, 7}), dst); + // TODO: add ignored Java test when it can be ignored + // IgnoreJava ignore_java(dst); + // generate_op2_join("hamming(a,b)", Seq({-128, -43, -1, 0, 85, 127}), ignore_java); // TODO: require java // inverted lambda generate_join_expr("join(a,b,f(a,b)(b-a))", Div16(N()), dst); // custom lambda @@ -331,6 +347,9 @@ void generate_merge(TestBuilder &dst) { generate_op2_merge("min(a,b)", Div16(N()), dst); generate_op2_merge("max(a,b)", Div16(N()), dst); generate_op2_merge("bit(a,b)", Seq({-128, -43, -1, 0, 85, 127}), Seq({0, 1, 2, 3, 4, 5, 6, 7}), dst); + // TODO: add ignored Java test when it can be ignored + // IgnoreJava ignore_java(dst); + // generate_op2_merge("hamming(a,b)", Seq({-128, -43, -1, 0, 85, 127}), ignore_java); // TODO: require java // inverted lambda generate_merge_expr("merge(a,b,f(a,b)(b-a))", Div16(N()), dst); // custom lambda diff --git a/eval/src/apps/tensor_conformance/generate.h b/eval/src/apps/tensor_conformance/generate.h index e9482b9015c..9aa90ae9a7a 100644 --- a/eval/src/apps/tensor_conformance/generate.h +++ b/eval/src/apps/tensor_conformance/generate.h @@ -18,11 +18,6 @@ struct TestBuilder { { add(expression, inputs, {}); } - void add_ignore_java(const vespalib::string &expression, - const std::map<vespalib::string,TensorSpec> &inputs) - { - add(expression, inputs, {"vespajlib"}); - } virtual ~TestBuilder() {} }; diff --git a/eval/src/apps/tensor_conformance/tensor_conformance.cpp b/eval/src/apps/tensor_conformance/tensor_conformance.cpp index e6bbb1f8a41..6c28b1e652e 100644 --- a/eval/src/apps/tensor_conformance/tensor_conformance.cpp +++ b/eval/src/apps/tensor_conformance/tensor_conformance.cpp @@ -167,6 +167,15 @@ void print_test(const Inspector &test, OutputWriter &dst) { } auto result = eval_expr(test, prod_factory); dst.printf("result: %s\n", result.to_string().c_str()); + auto ignore = extract_fields(test["ignore"]); + if (!ignore.empty()) { + dst.printf("ignore:"); + for (const auto &impl: ignore) { + REQUIRE(test["ignore"][impl].asBool()); + dst.printf(" %s", impl.c_str()); + } + dst.printf("\n"); + } } //----------------------------------------------------------------------------- diff --git a/eval/src/tests/eval/inline_operation/inline_operation_test.cpp b/eval/src/tests/eval/inline_operation/inline_operation_test.cpp index ae5f503b680..8e765708574 100644 --- a/eval/src/tests/eval/inline_operation/inline_operation_test.cpp +++ b/eval/src/tests/eval/inline_operation/inline_operation_test.cpp @@ -116,6 +116,7 @@ TEST(InlineOperationTest, op2_lambdas_are_recognized) { EXPECT_EQ(as_op2("min(a,b)"), &Min::f); EXPECT_EQ(as_op2("max(a,b)"), &Max::f); EXPECT_EQ(as_op2("bit(a,b)"), &Bit::f); + EXPECT_EQ(as_op2("hamming(a,b)"), &Hamming::f); } TEST(InlineOperationTest, op2_lambdas_are_recognized_with_different_parameter_names) { diff --git a/eval/src/tests/eval/node_tools/node_tools_test.cpp b/eval/src/tests/eval/node_tools/node_tools_test.cpp index e8296c01d73..b95ea2d4b14 100644 --- a/eval/src/tests/eval/node_tools/node_tools_test.cpp +++ b/eval/src/tests/eval/node_tools/node_tools_test.cpp @@ -101,6 +101,7 @@ TEST("require that call node types can be copied") { TEST_DO(verify_copy("elu(a)")); TEST_DO(verify_copy("erf(a)")); TEST_DO(verify_copy("bit(a,b)")); + TEST_DO(verify_copy("hamming(a,b)")); } TEST("require that tensor node types can NOT be copied (yet)") { diff --git a/eval/src/tests/eval/node_types/node_types_test.cpp b/eval/src/tests/eval/node_types/node_types_test.cpp index b2373f0d8f5..5b860f0e1b3 100644 --- a/eval/src/tests/eval/node_types/node_types_test.cpp +++ b/eval/src/tests/eval/node_types/node_types_test.cpp @@ -219,6 +219,7 @@ TEST("require that various operations resolve appropriate type") { TEST_DO(verify_op1("elu(%s)")); // Elu TEST_DO(verify_op1("erf(%s)")); // Erf TEST_DO(verify_op2("bit(%s,%s)")); // Bit + TEST_DO(verify_op2("hamming(%s,%s)")); // Hamming } TEST("require that map resolves correct type") { diff --git a/eval/src/tests/instruction/dense_hamming_distance/CMakeLists.txt b/eval/src/tests/instruction/dense_hamming_distance/CMakeLists.txt new file mode 100644 index 00000000000..3d18f9613b3 --- /dev/null +++ b/eval/src/tests/instruction/dense_hamming_distance/CMakeLists.txt @@ -0,0 +1,9 @@ + +vespa_add_executable(eval_dense_hamming_distance_test_app TEST + SOURCES + dense_hamming_distance_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_dense_hamming_distance_test_app COMMAND eval_dense_hamming_distance_test_app) diff --git a/eval/src/tests/instruction/dense_hamming_distance/dense_hamming_distance_test.cpp b/eval/src/tests/instruction/dense_hamming_distance/dense_hamming_distance_test.cpp new file mode 100644 index 00000000000..8eaa0e72ad5 --- /dev/null +++ b/eval/src/tests/instruction/dense_hamming_distance/dense_hamming_distance_test.cpp @@ -0,0 +1,91 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/tensor_function.h> +#include <vespa/eval/eval/test/eval_fixture.h> +#include <vespa/eval/eval/test/gen_spec.h> +#include <vespa/eval/instruction/dense_hamming_distance.h> +#include <vespa/vespalib/util/stash.h> +#include <vespa/vespalib/util/stringfmt.h> + +#include <vespa/vespalib/util/require.h> +#include <vespa/vespalib/gtest/gtest.h> + +#include <vespa/log/log.h> +LOG_SETUP("dense_hamming_distance_function_test"); + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::eval::test; + +const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get(); + +struct FunInfo { + using LookFor = DenseHammingDistance; + void verify(const LookFor &fun) const { + EXPECT_TRUE(fun.result_is_mutable()); + } +}; + +void assertOptimized(const vespalib::string &expr) { + CellTypeSpace just_int8({CellType::INT8}, 2); + EvalFixture::verify<FunInfo>(expr, {FunInfo{}}, just_int8); + CellTypeSpace just_double({CellType::DOUBLE}, 2); + EvalFixture::verify<FunInfo>(expr, {}, just_double); +} + +void assertNotOptimized(const vespalib::string &expr) { + CellTypeSpace just_int8({CellType::INT8}, 2); + EvalFixture::verify<FunInfo>(expr, {}, just_int8); +} + +TEST(DenseHammingDistanceOptimizer, hamming_distance_works_with_tensor_function) { + assertOptimized("reduce(hamming(x5$1,x5$2),sum)"); + assertOptimized("reduce(hamming(x5$1,x5$2),sum,x)"); + assertOptimized("reduce(join(x5$1,x5$2,f(x,y)(hamming(x,y))),sum)"); + assertOptimized("reduce(join(x5$1,x5$2,f(x,y)(hamming(x,y))),sum,x)"); +} + +TEST(DenseHammingDistanceOptimizer, hamming_distance_with_compatible_dimensions_is_optimized) { + // various vector sizes + assertOptimized("reduce(hamming(x1$1,x1$2),sum)"); + assertOptimized("reduce(hamming(x3$1,x3$2),sum)"); + assertOptimized("reduce(hamming(x7$1,x7$2),sum)"); + assertOptimized("reduce(hamming(x8$1,x8$2),sum)"); + assertOptimized("reduce(hamming(x9$1,x9$2),sum)"); + assertOptimized("reduce(hamming(x17$1,x17$2),sum)"); + // multiple dimensions + assertOptimized("reduce(hamming(x3y3$1,x3y3$2),sum)"); + assertOptimized("reduce(hamming(x3y4$1,x3y4$2),sum)"); + // with trivial dimensions + assertOptimized("reduce(hamming(a1x3$1,x3$2),sum)"); + assertOptimized("reduce(hamming(x3$1z1,x3$2),sum)"); + assertOptimized("reduce(hamming(a1x3$1,b1x3$2z1),sum)"); +} + +TEST(DenseHammingDistanceOptimizer, hamming_distance_with_mapped_dimensions_is_NOT_optimized) { + assertNotOptimized("reduce(hamming(x3_1$1,x3_1$2),sum)"); + assertNotOptimized("reduce(hamming(x3_1y2$1,x3_1y2$2),sum)"); +} + +TEST(DenseHammingDistanceOptimizer, hamming_distance_with_incompatible_dimensions_is_NOT_optimized) { + assertNotOptimized("reduce(hamming(x3,y3),sum)"); + assertNotOptimized("reduce(hamming(y3,x3),sum)"); + assertNotOptimized("reduce(hamming(x3,x3y3),sum)"); + assertNotOptimized("reduce(hamming(x3y3,x3),sum)"); +} + +TEST(DenseHammingDistanceOptimizer, expressions_similar_to_hamming_distance_are_not_optimized) { + assertNotOptimized("reduce(hamming(x3$1,x3$2),prod)"); +} + +TEST(DenseHammingDistanceOptimizer, result_must_be_double_to_trigger_optimization) { + assertOptimized("reduce(hamming(x3y3$1,x3y3$2),sum,x,y)"); + assertNotOptimized("reduce(hamming(x3y3$1,x3y3$2),sum,x)"); + assertNotOptimized("reduce(hamming(x3y3$1,x3y3$2),sum,y)"); +} + +//----------------------------------------------------------------------------- + +GTEST_MAIN_RUN_ALL_TESTS() + diff --git a/eval/src/vespa/eval/eval/call_nodes.cpp b/eval/src/vespa/eval/eval/call_nodes.cpp index 798583cf89a..95dbecdd153 100644 --- a/eval/src/vespa/eval/eval/call_nodes.cpp +++ b/eval/src/vespa/eval/eval/call_nodes.cpp @@ -44,6 +44,7 @@ CallRepo::CallRepo() : _map() { add(nodes::Elu()); add(nodes::Erf()); add(nodes::Bit()); + add(nodes::Hamming()); } } // namespace vespalib::eval::nodes diff --git a/eval/src/vespa/eval/eval/call_nodes.h b/eval/src/vespa/eval/eval/call_nodes.h index 945aba69596..47fc5d6eccd 100644 --- a/eval/src/vespa/eval/eval/call_nodes.h +++ b/eval/src/vespa/eval/eval/call_nodes.h @@ -140,6 +140,7 @@ struct Sigmoid : CallHelper<Sigmoid> { Sigmoid() : Helper("sigmoid", 1) {} }; struct Elu : CallHelper<Elu> { Elu() : Helper("elu", 1) {} }; struct Erf : CallHelper<Erf> { Erf() : Helper("erf", 1) {} }; struct Bit : CallHelper<Bit> { Bit() : Helper("bit", 2) {} }; +struct Hamming : CallHelper<Hamming> { Hamming() : Helper("hamming", 2) {} }; //----------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/eval/hamming_distance.h b/eval/src/vespa/eval/eval/hamming_distance.h new file mode 100644 index 00000000000..3419de3569f --- /dev/null +++ b/eval/src/vespa/eval/eval/hamming_distance.h @@ -0,0 +1,13 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace vespalib::eval { + +inline double hamming_distance(double a, double b) { + uint8_t x = (uint8_t) a; + uint8_t y = (uint8_t) b; + return __builtin_popcount(x ^ y); +} + +} diff --git a/eval/src/vespa/eval/eval/key_gen.cpp b/eval/src/vespa/eval/eval/key_gen.cpp index a40a8887119..cbbce61402c 100644 --- a/eval/src/vespa/eval/eval/key_gen.cpp +++ b/eval/src/vespa/eval/eval/key_gen.cpp @@ -88,6 +88,7 @@ struct KeyGen : public NodeVisitor, public NodeTraverser { void visit(const Elu &) override { add_byte(61); } void visit(const Erf &) override { add_byte(62); } void visit(const Bit &) override { add_byte(63); } + void visit(const Hamming &) override { add_byte(64); } // traverse bool open(const Node &node) override { node.accept(*this); return true; } diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp index 3e4f4fe8257..a101745dca0 100644 --- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp +++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp @@ -5,6 +5,7 @@ #include <vespa/eval/eval/node_visitor.h> #include <vespa/eval/eval/node_traverser.h> #include <vespa/eval/eval/extract_bit.h> +#include <vespa/eval/eval/hamming_distance.h> #include <llvm/IR/Verifier.h> #include <llvm/Support/TargetSelect.h> #include <llvm/IR/IRBuilder.h> @@ -31,6 +32,7 @@ double vespalib_eval_relu(double a) { return std::max(a, 0.0); } double vespalib_eval_sigmoid(double a) { return 1.0 / (1.0 + std::exp(-1.0 * a)); } double vespalib_eval_elu(double a) { return (a < 0) ? std::exp(a) - 1.0 : a; } double vespalib_eval_bit(double a, double b) { return vespalib::eval::extract_bit(a, b); } +double vespalib_eval_hamming(double a, double b) { return vespalib::eval::hamming_distance(a, b); } using vespalib::eval::gbdt::Forest; using resolve_function = double (*)(void *ctx, size_t idx); @@ -651,6 +653,9 @@ struct FunctionBuilder : public NodeVisitor, public NodeTraverser { void visit(const Bit &) override { make_call_2("vespalib_eval_bit"); } + void visit(const Hamming &) override { + make_call_2("vespalib_eval_hamming"); + } }; FunctionBuilder::~FunctionBuilder() { } diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h index e04b477750d..727954d59e9 100644 --- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h +++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h @@ -20,6 +20,7 @@ extern "C" { double vespalib_eval_sigmoid(double a); double vespalib_eval_elu(double a); double vespalib_eval_bit(double a, double b); + double vespalib_eval_hamming(double a, double b); }; namespace vespalib::eval { diff --git a/eval/src/vespa/eval/eval/make_tensor_function.cpp b/eval/src/vespa/eval/eval/make_tensor_function.cpp index 498be2a738b..7746676f86b 100644 --- a/eval/src/vespa/eval/eval/make_tensor_function.cpp +++ b/eval/src/vespa/eval/eval/make_tensor_function.cpp @@ -360,6 +360,9 @@ struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser { void visit(const Bit &node) override { make_join(node, operation::Bit::f); } + void visit(const Hamming &node) override { + make_join(node, operation::Hamming::f); + } //------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/eval/node_tools.cpp b/eval/src/vespa/eval/eval/node_tools.cpp index fa2d16a2271..48ee1a90b67 100644 --- a/eval/src/vespa/eval/eval/node_tools.cpp +++ b/eval/src/vespa/eval/eval/node_tools.cpp @@ -183,6 +183,7 @@ struct CopyNode : NodeTraverser, NodeVisitor { void visit(const Elu &node) override { copy_call(node); } void visit(const Erf &node) override { copy_call(node); } void visit(const Bit &node) override { copy_call(node); } + void visit(const Hamming &node) override { copy_call(node); } // traverse nodes bool open(const Node &) override { return !error; } diff --git a/eval/src/vespa/eval/eval/node_types.cpp b/eval/src/vespa/eval/eval/node_types.cpp index 8622fd734f1..2cb6e637201 100644 --- a/eval/src/vespa/eval/eval/node_types.cpp +++ b/eval/src/vespa/eval/eval/node_types.cpp @@ -279,6 +279,7 @@ struct TypeResolver : public NodeVisitor, public NodeTraverser { void visit(const Elu &node) override { resolve_op1(node); } void visit(const Erf &node) override { resolve_op1(node); } void visit(const Bit &node) override { resolve_op2(node); } + void visit(const Hamming &node) override { resolve_op2(node); } //------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/eval/node_visitor.h b/eval/src/vespa/eval/eval/node_visitor.h index 475bbf5405c..b581a94f7ee 100644 --- a/eval/src/vespa/eval/eval/node_visitor.h +++ b/eval/src/vespa/eval/eval/node_visitor.h @@ -86,6 +86,7 @@ struct NodeVisitor { virtual void visit(const nodes::Elu &) = 0; virtual void visit(const nodes::Erf &) = 0; virtual void visit(const nodes::Bit &) = 0; + virtual void visit(const nodes::Hamming &) = 0; virtual ~NodeVisitor() {} }; @@ -156,6 +157,7 @@ struct EmptyNodeVisitor : NodeVisitor { void visit(const nodes::Elu &) override {} void visit(const nodes::Erf &) override {} void visit(const nodes::Bit &) override {} + void visit(const nodes::Hamming &) override {} }; } // namespace vespalib::eval diff --git a/eval/src/vespa/eval/eval/operation.cpp b/eval/src/vespa/eval/eval/operation.cpp index a82a79e6bc4..ddd188d250f 100644 --- a/eval/src/vespa/eval/eval/operation.cpp +++ b/eval/src/vespa/eval/eval/operation.cpp @@ -4,6 +4,7 @@ #include "function.h" #include "key_gen.h" #include "extract_bit.h" +#include "hamming_distance.h" #include <vespa/vespalib/util/approx.h> #include <algorithm> @@ -52,6 +53,7 @@ double Sigmoid::f(double a) { return 1.0 / (1.0 + std::exp(-1.0 * a)); } double Elu::f(double a) { return (a < 0) ? std::exp(a) - 1 : a; } double Erf::f(double a) { return std::erf(a); } double Bit::f(double a, double b) { return extract_bit(a, b); } +double Hamming::f(double a, double b) { return hamming_distance(a, b); } //----------------------------------------------------------------------------- double Inv::f(double a) { return (1.0 / a); } double Square::f(double a) { return (a * a); } @@ -146,6 +148,7 @@ std::map<vespalib::string,op2_t> make_op2_map() { add_op2(map, "min(a,b)", Min::f); add_op2(map, "max(a,b)", Max::f); add_op2(map, "bit(a,b)", Bit::f); + add_op2(map, "hamming(a,b)", Hamming::f); return map; } diff --git a/eval/src/vespa/eval/eval/operation.h b/eval/src/vespa/eval/eval/operation.h index 438b510b714..e2a524f318c 100644 --- a/eval/src/vespa/eval/eval/operation.h +++ b/eval/src/vespa/eval/eval/operation.h @@ -50,6 +50,7 @@ struct Sigmoid { static double f(double a); }; struct Elu { static double f(double a); }; struct Erf { static double f(double a); }; struct Bit { static double f(double a, double b); }; +struct Hamming { static double f(double a, double b); }; //----------------------------------------------------------------------------- struct Inv { static double f(double a); }; struct Square { static double f(double a); }; diff --git a/eval/src/vespa/eval/eval/optimize_tensor_function.cpp b/eval/src/vespa/eval/eval/optimize_tensor_function.cpp index 64acbceff04..c2e8d886fde 100644 --- a/eval/src/vespa/eval/eval/optimize_tensor_function.cpp +++ b/eval/src/vespa/eval/eval/optimize_tensor_function.cpp @@ -28,6 +28,7 @@ #include <vespa/eval/instruction/vector_from_doubles_function.h> #include <vespa/eval/instruction/dense_tensor_create_function.h> #include <vespa/eval/instruction/dense_tensor_peek_function.h> +#include <vespa/eval/instruction/dense_hamming_distance.h> #include <vespa/log/log.h> LOG_SETUP(".eval.eval.optimize_tensor_function"); @@ -53,6 +54,7 @@ const TensorFunction &optimize_for_factory(const ValueBuilderFactory &, const Te child.set(DenseMatMulFunction::optimize(child.get(), stash)); child.set(DenseMultiMatMulFunction::optimize(child.get(), stash)); child.set(MixedInnerProductFunction::optimize(child.get(), stash)); + child.set(DenseHammingDistance::optimize(child.get(), stash)); nodes.pop_back(); } } diff --git a/eval/src/vespa/eval/eval/test/eval_spec.cpp b/eval/src/vespa/eval/eval/test/eval_spec.cpp index 5d51a1d23b5..03b3af84fc9 100644 --- a/eval/src/vespa/eval/eval/test/eval_spec.cpp +++ b/eval/src/vespa/eval/eval/test/eval_spec.cpp @@ -8,6 +8,24 @@ namespace vespalib::eval::test { +namespace { + +double byte(const vespalib::string &bits) { + int8_t res = 0; + assert(bits.size() == 8); + for (const auto &c: bits) { + if (c == '1') { + res = (res << 1) | 1; + } else { + assert(c == '0'); + res = (res << 1); + } + } + return res; +} + +} // <unnamed> + constexpr double my_nan = std::numeric_limits<double>::quiet_NaN(); constexpr double my_inf = std::numeric_limits<double>::infinity(); @@ -169,6 +187,9 @@ EvalSpec::add_function_call_cases() { .add_case({85, 3}, 0.0).add_case({85, 2}, 1.0).add_case({85, 1}, 0.0).add_case({85, 0}, 1.0) .add_case({127, 7}, 0.0).add_case({127, 6}, 1.0).add_case({127, 5}, 1.0).add_case({127, 4}, 1.0) .add_case({127, 3}, 1.0).add_case({127, 2}, 1.0).add_case({127, 1}, 1.0).add_case({127, 0}, 1.0); + add_expression({"a", "b"}, "hamming(a,b)") + .add_case({0, 0}, 0.0).add_case({-1, -1}, 0.0).add_case({-1, 0}, 8.0).add_case({0, -1}, 8.0) + .add_case({byte("11001100"), byte("10101010")}, 4.0).add_case({byte("11001100"), byte("11110000")}, 4.0); } void diff --git a/eval/src/vespa/eval/eval/test/reference_evaluation.cpp b/eval/src/vespa/eval/eval/test/reference_evaluation.cpp index 58e4b91f6d9..def3f64c1a1 100644 --- a/eval/src/vespa/eval/eval/test/reference_evaluation.cpp +++ b/eval/src/vespa/eval/eval/test/reference_evaluation.cpp @@ -338,6 +338,9 @@ struct EvalNode : public NodeVisitor { void visit(const Bit &node) override { eval_join(node.get_child(0), node.get_child(1), operation::Bit::f); } + void visit(const Hamming &node) override { + eval_join(node.get_child(0), node.get_child(1), operation::Hamming::f); + } }; TensorSpec eval_node(const Node &node, const std::vector<TensorSpec> ¶ms) { diff --git a/eval/src/vespa/eval/eval/visit_stuff.cpp b/eval/src/vespa/eval/eval/visit_stuff.cpp index 786562d823f..1d684e1c340 100644 --- a/eval/src/vespa/eval/eval/visit_stuff.cpp +++ b/eval/src/vespa/eval/eval/visit_stuff.cpp @@ -60,6 +60,7 @@ vespalib::string name_of(join_fun_t fun) { if (fun == operation::Min::f) return "min"; if (fun == operation::Max::f) return "max"; if (fun == operation::Bit::f) return "bit"; + if (fun == operation::Hamming::f) return "hamming"; return "[other join function]"; } diff --git a/eval/src/vespa/eval/instruction/CMakeLists.txt b/eval/src/vespa/eval/instruction/CMakeLists.txt index 88e3272bb7c..3ed969c0a18 100644 --- a/eval/src/vespa/eval/instruction/CMakeLists.txt +++ b/eval/src/vespa/eval/instruction/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_library(eval_instruction OBJECT add_trivial_dimension_optimizer.cpp dense_cell_range_function.cpp dense_dot_product_function.cpp + dense_hamming_distance.cpp dense_lambda_peek_function.cpp dense_lambda_peek_optimizer.cpp dense_matmul_function.cpp diff --git a/eval/src/vespa/eval/instruction/dense_hamming_distance.cpp b/eval/src/vespa/eval/instruction/dense_hamming_distance.cpp new file mode 100644 index 00000000000..2a68663631a --- /dev/null +++ b/eval/src/vespa/eval/instruction/dense_hamming_distance.cpp @@ -0,0 +1,91 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "dense_hamming_distance.h" +#include <vespa/eval/eval/operation.h> +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/hamming_distance.h> + +#include <vespa/log/log.h> +LOG_SETUP(".eval.instruction.dense_hamming_distance"); + +namespace vespalib::eval { + +using namespace tensor_function; + +namespace { + + +size_t binary_hamming_distance(const void *lhs, const void *rhs, size_t sz) { + const uint64_t *words_a = static_cast<const uint64_t *>(lhs); + const uint64_t *words_b = static_cast<const uint64_t *>(rhs); + size_t sum = 0; + size_t i = 0; + for (; i * 8 + 7 < sz; ++i) { + uint64_t xor_bits = words_a[i] ^ words_b[i]; + sum += __builtin_popcountl(xor_bits); + } + if (__builtin_expect((i * 8 < sz), false)) { + const uint8_t *bytes_a = static_cast<const uint8_t *>(lhs); + const uint8_t *bytes_b = static_cast<const uint8_t *>(rhs); + for (i *= 8; i < sz; ++i) { + uint64_t xor_bits = bytes_a[i] ^ bytes_b[i]; + sum += __builtin_popcountl(xor_bits); + } + } + return sum; +}; + +void int8_hamming_to_double_op(InterpretedFunction::State &state, uint64_t vector_size) { + const auto &lhs = state.peek(1); + const auto &rhs = state.peek(0); + auto a = lhs.cells(); + auto b = rhs.cells(); + double result = binary_hamming_distance(a.data, b.data, vector_size); + state.pop_pop_push(state.stash.create<DoubleValue>(result)); +} + +bool compatible_types(const ValueType &lhs, const ValueType &rhs) { + return ((lhs.cell_type() == CellType::INT8) && + (rhs.cell_type() == CellType::INT8) && + lhs.is_dense() && + rhs.is_dense() && + (lhs.nontrivial_indexed_dimensions() == rhs.nontrivial_indexed_dimensions())); +} + +} // namespace <unnamed> + +DenseHammingDistance::DenseHammingDistance(const TensorFunction &lhs_child, + const TensorFunction &rhs_child) + : tensor_function::Op2(ValueType::double_type(), lhs_child, rhs_child) +{ +} + +InterpretedFunction::Instruction +DenseHammingDistance::compile_self(const ValueBuilderFactory &, Stash &) const +{ + auto op = int8_hamming_to_double_op; + const auto &lhs_type = lhs().result_type(); + const auto &rhs_type = rhs().result_type(); + LOG_ASSERT(lhs_type.dense_subspace_size() == rhs_type.dense_subspace_size()); + return InterpretedFunction::Instruction(op, lhs_type.dense_subspace_size()); +} + +const TensorFunction & +DenseHammingDistance::optimize(const TensorFunction &expr, Stash &stash) +{ + const auto & res_type = expr.result_type(); + auto reduce = as<Reduce>(expr); + if (res_type.is_double() && reduce && (reduce->aggr() == Aggr::SUM)) { + auto join = as<Join>(reduce->child()); + if (join && (join->function() == operation::Hamming::f)) { + const TensorFunction &lhs = join->lhs(); + const TensorFunction &rhs = join->rhs(); + if (compatible_types(lhs.result_type(), rhs.result_type())) { + return stash.create<DenseHammingDistance>(lhs, rhs); + } + } + } + return expr; +} + +} // namespace diff --git a/eval/src/vespa/eval/instruction/dense_hamming_distance.h b/eval/src/vespa/eval/instruction/dense_hamming_distance.h new file mode 100644 index 00000000000..efc70d74d21 --- /dev/null +++ b/eval/src/vespa/eval/instruction/dense_hamming_distance.h @@ -0,0 +1,22 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/tensor_function.h> + +namespace vespalib::eval { + +/** + * Tensor function for a hamming distance producing a scalar result. + **/ +class DenseHammingDistance : public tensor_function::Op2 +{ +public: + DenseHammingDistance(const TensorFunction &lhs_child, + const TensorFunction &rhs_child); + InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + bool result_is_mutable() const override { return true; } + static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); +}; + +} // namespace diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java index ede7bd6a109..5b3b2a94beb 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java @@ -4,9 +4,7 @@ package com.yahoo.vespa.flags; import com.yahoo.vespa.flags.json.DimensionHelper; import javax.annotation.concurrent.Immutable; -import java.util.Collections; import java.util.EnumMap; -import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -72,15 +70,15 @@ public class FetchVector { private final Map<Dimension, String> map; public FetchVector() { - this.map = Collections.emptyMap(); + this.map = Map.of(); } public static FetchVector fromMap(Map<Dimension, String> map) { - return new FetchVector(new HashMap<>(map)); + return new FetchVector(map); } private FetchVector(Map<Dimension, String> map) { - this.map = Collections.unmodifiableMap(map); + this.map = Map.copyOf(map); } public Optional<String> getValue(Dimension dimension) { @@ -93,6 +91,10 @@ public class FetchVector { public boolean isEmpty() { return map.isEmpty(); } + public boolean hasDimension(FetchVector.Dimension dimension) { + return map.containsKey(dimension); + } + /** * Returns a new FetchVector, identical to {@code this} except for its value in {@code dimension}. * Dimension is removed if the value is null. diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FlagDefinition.java b/flags/src/main/java/com/yahoo/vespa/flags/FlagDefinition.java index d01ca64cb9f..7ddbd85a904 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FlagDefinition.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FlagDefinition.java @@ -3,9 +3,9 @@ package com.yahoo.vespa.flags; import javax.annotation.concurrent.Immutable; import java.time.Instant; -import java.util.Arrays; -import java.util.Collections; +import java.util.EnumSet; import java.util.List; +import java.util.Set; /** * @author hakonhall @@ -28,14 +28,14 @@ public class FlagDefinition { String description, String modificationEffect, FetchVector.Dimension... dimensions) { - validate(owners, createdAt, expiresAt); this.unboundFlag = unboundFlag; this.owners = owners; this.createdAt = createdAt; this.expiresAt = expiresAt; this.description = description; this.modificationEffect = modificationEffect; - this.dimensions = Collections.unmodifiableList(Arrays.asList(dimensions)); + this.dimensions = List.of(dimensions); + validate(owners, createdAt, expiresAt, this.dimensions); } public UnboundFlag<?, ?, ?> getUnboundFlag() { @@ -60,13 +60,14 @@ public class FlagDefinition { public Instant getExpiresAt() { return expiresAt; } - private static void validate(List<String> owners, Instant createdAt, Instant expiresAt) { + private static void validate(List<String> owners, Instant createdAt, Instant expiresAt, List<FetchVector.Dimension> dimensions) { if (expiresAt.isBefore(createdAt)) { throw new IllegalArgumentException( String.format( "Flag cannot expire before its creation date (createdAt='%s', expiresAt='%s')", createdAt, expiresAt)); } + if (owners == PermanentFlags.OWNERS) { if (!createdAt.equals(PermanentFlags.CREATED_AT) || !expiresAt.equals(PermanentFlags.EXPIRES_AT)) { throw new IllegalArgumentException("Invalid creation or expiration date for permanent flag"); @@ -74,5 +75,15 @@ public class FlagDefinition { } else if (owners.isEmpty()) { throw new IllegalArgumentException("Owner(s) must be specified"); } + + if (dimensions.contains(FetchVector.Dimension.CONSOLE_USER_EMAIL)) { + Set<FetchVector.Dimension> disallowedCombinations = EnumSet.allOf(FetchVector.Dimension.class); + disallowedCombinations.remove(FetchVector.Dimension.CONSOLE_USER_EMAIL); + disallowedCombinations.remove(FetchVector.Dimension.APPLICATION_ID); + disallowedCombinations.remove(FetchVector.Dimension.TENANT_ID); + disallowedCombinations.retainAll(dimensions); + if (!disallowedCombinations.isEmpty()) + throw new IllegalArgumentException("Dimension " + FetchVector.Dimension.CONSOLE_USER_EMAIL + " cannot be combined with " + disallowedCombinations); + } } } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 015e063fe11..87736d8e591 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -13,6 +13,7 @@ import java.util.Optional; import java.util.TreeMap; import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; +import static com.yahoo.vespa.flags.FetchVector.Dimension.CONSOLE_USER_EMAIL; import static com.yahoo.vespa.flags.FetchVector.Dimension.HOSTNAME; import static com.yahoo.vespa.flags.FetchVector.Dimension.TENANT_ID; import static com.yahoo.vespa.flags.FetchVector.Dimension.VESPA_VERSION; @@ -78,13 +79,6 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag ENFORCE_RANK_PROFILE_INHERITANCE = defineFeatureFlag( - "enforce-rank-profile-inheritance", false, - List.of("baldersheim"), "2021-09-07", "2021-10-01", - "Should we enforce verification of rank-profile inheritance.", - "Takes effect at redeployment", - ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag SKIP_MBUS_REQUEST_THREAD = defineFeatureFlag( "skip-mbus-request-thread", false, List.of("baldersheim"), "2020-12-02", "2022-01-01", @@ -135,7 +129,7 @@ public class Flags { "Takes effect on first (re)start of config server"); public static final UnboundBooleanFlag ENCRYPT_DIRTY_DISK = defineFeatureFlag( - "encrypt-dirty-disk", false, + "encrypt-dirty-disk", true, List.of("hakonhall"), "2021-05-14", "2021-10-05", "Allow migrating an unencrypted data partition to being encrypted when (de)provisioned.", "Takes effect on next host-admin tick."); @@ -146,14 +140,8 @@ public class Flags { "Only allow modifications of disks by disk task in limited situations.", "Takes effect on next host-admin tick."); - public static final UnboundBooleanFlag NEW_SPARE_DISKS = defineFeatureFlag( - "new-spare-disks", true, - List.of("hakonhall"), "2021-09-08", "2021-11-08", - "Use a new algorithm to calculate the spare disks of a host.", - "Takes effect on first run of DiskTask, typically after host-admin restart/upgrade."); - public static final UnboundBooleanFlag LOCAL_SUSPEND = defineFeatureFlag( - "local-suspend", true, + "local-suspend", false, List.of("hakonhall"), "2021-09-21", "2021-10-21", "Whether the cfghost host admin should suspend against only the local cfg (true and legacy) or all.", "Takes effect immediately."); @@ -171,6 +159,20 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag CONTAINER_DUMP_HEAP_ON_SHUTDOWN_TIMEOUT = defineFeatureFlag( + "container-dump-heap-on-shutdown-timeout", false, + List.of("baldersheim"), "2021-09-25", "2021-11-01", + "Will trigger a heap dump during if container shutdown times out", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + + public static final UnboundDoubleFlag CONTAINER_SHUTDOWN_TIMEOUT = defineDoubleFlag( + "container-shutdown-timeout", 50.0, + List.of("baldersheim"), "2021-09-25", "2021-11-01", + "Timeout for shutdown of a jdisc container", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundListFlag<String> ALLOWED_ATHENZ_PROXY_IDENTITIES = defineListFlag( "allowed-athenz-proxy-identities", List.of(), String.class, List.of("bjorncs", "tokle"), "2021-02-10", "2021-12-01", @@ -278,7 +280,7 @@ public class Flags { List.of("olaa"), "2021-09-13", "2021-12-31", "Enable Horizon dashboard", "Takes effect immediately", - TENANT_ID + TENANT_ID, CONSOLE_USER_EMAIL ); public static final UnboundBooleanFlag ENABLE_ONPREM_TENANT_S3_ARCHIVE = defineFeatureFlag( @@ -297,6 +299,15 @@ public class Flags { APPLICATION_ID ); + public static final UnboundBooleanFlag ENABLE_TENANT_DEVELOPER_ROLE = defineFeatureFlag( + "enable-tenant-developer-role", false, + List.of("bjorncs"), "2021-09-23", "2021-12-31", + "Enable tenant developer Athenz role in cd/main. Must be set on controller cluster only.", + "Takes effect immediately", + TENANT_ID + ); + + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, @@ -416,8 +427,8 @@ public class Flags { * * <p>NOT thread-safe. Tests using this cannot run in parallel. */ - public static Replacer clearFlagsForTesting() { - return new Replacer(); + public static Replacer clearFlagsForTesting(FlagId... flagsToKeep) { + return new Replacer(flagsToKeep); } public static class Replacer implements AutoCloseable { @@ -425,10 +436,11 @@ public class Flags { private final TreeMap<FlagId, FlagDefinition> savedFlags; - private Replacer() { + private Replacer(FlagId... flagsToKeep) { verifyAndSetFlagsCleared(true); this.savedFlags = Flags.flags; Flags.flags = new TreeMap<>(); + List.of(flagsToKeep).forEach(id -> Flags.flags.put(id, savedFlags.get(id))); } @Override diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/Condition.java b/flags/src/main/java/com/yahoo/vespa/flags/json/Condition.java index 46961fbd8cc..f73e0033773 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/Condition.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/Condition.java @@ -52,6 +52,16 @@ public interface Condition extends Predicate<FetchVector> { public FetchVector.Dimension dimension() { return dimension; } public List<String> values() { return values; } public Optional<String> predicate() { return predicate; } + + public Condition createAs(Condition.Type type) { + switch (type) { + case WHITELIST: return WhitelistCondition.create(this); + case BLACKLIST: return BlacklistCondition.create(this); + case RELATIONAL: return RelationalCondition.create(this); + } + + throw new IllegalArgumentException("Unknown type '" + type + "'"); + } } static Condition fromWire(WireCondition wireCondition) { @@ -70,14 +80,14 @@ public interface Condition extends Predicate<FetchVector> { params.withPredicate(wireCondition.predicate); } - switch (type) { - case WHITELIST: return WhitelistCondition.create(params); - case BLACKLIST: return BlacklistCondition.create(params); - case RELATIONAL: return RelationalCondition.create(params); - } - - throw new IllegalArgumentException("Unknown type '" + type + "'"); + return params.createAs(type); } + Condition.Type type(); + + FetchVector.Dimension dimension(); + + CreateParams toCreateParams(); + WireCondition toWire(); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java b/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java index c4079380a8c..eea61eb71ef 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java @@ -14,9 +14,6 @@ import com.yahoo.vespa.flags.json.wire.WireRule; import javax.annotation.concurrent.Immutable; import java.io.InputStream; import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -35,16 +32,16 @@ public class FlagData { private final FetchVector defaultFetchVector; public FlagData(FlagId id) { - this(id, new FetchVector(), Collections.emptyList()); + this(id, new FetchVector(), List.of()); } public FlagData(FlagId id, FetchVector defaultFetchVector, Rule... rules) { - this(id, defaultFetchVector, Arrays.asList(rules)); + this(id, defaultFetchVector, List.of(rules)); } public FlagData(FlagId id, FetchVector defaultFetchVector, List<Rule> rules) { this.id = id; - this.rules = Collections.unmodifiableList(new ArrayList<>(rules)); + this.rules = List.copyOf(rules); this.defaultFetchVector = defaultFetchVector; } @@ -52,6 +49,10 @@ public class FlagData { return id; } + public List<Rule> rules() { + return rules; + } + public boolean isEmpty() { return rules.isEmpty() && defaultFetchVector.isEmpty(); } public Optional<RawFlag> resolve(FetchVector fetchVector) { @@ -136,7 +137,7 @@ public class FlagData { } private static List<Rule> rulesFromWire(List<WireRule> wireRules) { - if (wireRules == null) return Collections.emptyList(); + if (wireRules == null) return List.of(); return wireRules.stream().map(Rule::fromWire).collect(Collectors.toList()); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/ListCondition.java b/flags/src/main/java/com/yahoo/vespa/flags/json/ListCondition.java index c2c76529833..136857bea5f 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/ListCondition.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/ListCondition.java @@ -27,6 +27,21 @@ public abstract class ListCondition implements Condition { } @Override + public Type type() { + return type; + } + + @Override + public FetchVector.Dimension dimension() { + return dimension; + } + + @Override + public CreateParams toCreateParams() { + return new CreateParams(dimension).withValues(values); + } + + @Override public boolean test(FetchVector fetchVector) { boolean listContainsValue = fetchVector.getValue(dimension).map(values::contains).orElse(false); return isWhitelist == listContainsValue; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java b/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java index db2f0a3a197..4ed3e49029f 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java @@ -48,6 +48,21 @@ public class RelationalCondition implements Condition { } @Override + public Type type() { + return Type.RELATIONAL; + } + + @Override + public FetchVector.Dimension dimension() { + return dimension; + } + + @Override + public CreateParams toCreateParams() { + return new CreateParams(dimension).withPredicate(relationalPredicate.toWire()); + } + + @Override public boolean test(FetchVector fetchVector) { return fetchVector.getValue(dimension).map(predicate::test).orElse(false); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/Rule.java b/flags/src/main/java/com/yahoo/vespa/flags/json/Rule.java index b7d60889419..0d50f1e283f 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/Rule.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/Rule.java @@ -6,7 +6,6 @@ import com.yahoo.vespa.flags.JsonNodeRawFlag; import com.yahoo.vespa.flags.RawFlag; import com.yahoo.vespa.flags.json.wire.WireRule; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -20,18 +19,32 @@ public class Rule { private final Optional<RawFlag> valueToApply; public Rule(Optional<RawFlag> valueToApply, Condition... andConditions) { - this(valueToApply, Arrays.asList(andConditions)); + this(valueToApply, List.of(andConditions)); } public Rule(Optional<RawFlag> valueToApply, List<Condition> andConditions) { - this.andConditions = andConditions; + this.andConditions = List.copyOf(andConditions); this.valueToApply = valueToApply; } + public List<Condition> conditions() { + return andConditions; + } + + /** Returns true if all the conditions satisfy the given fetch vector */ public boolean match(FetchVector fetchVector) { return andConditions.stream().allMatch(condition -> condition.test(fetchVector)); } + /** + * Returns true if all the conditions on dimensions set in the fetch vector are satisfied. + * Conditions on dimensions not specified in the given fetch vector are ignored. + */ + public boolean partialMatch(FetchVector fetchVector) { + return andConditions.stream() + .allMatch(condition -> !fetchVector.hasDimension(condition.dimension()) || condition.test(fetchVector)); + } + public Optional<RawFlag> getValueToApply() { return valueToApply; } diff --git a/functions.cmake b/functions.cmake index fe59cc3aaa9..3d192552be5 100644 --- a/functions.cmake +++ b/functions.cmake @@ -125,17 +125,13 @@ function(vespa_generate_config TARGET RELATIVE_CONFIG_DEF_PATH) get_filename_component(CONFIG_NAME ${RELATIVE_CONFIG_DEF_PATH} NAME_WE) endif() - # configgen.jar takes the parent dir of the destination dir and the destination dirname as separate parameters - # so it can produce the correct include statements within the generated .cpp-file (silent cry) # Make config path an absolute_path set(CONFIG_DEF_PATH ${CMAKE_CURRENT_LIST_DIR}/${RELATIVE_CONFIG_DEF_PATH}) - # Config destination is the + # Config destination is the current source directory (or parallel in build tree) + # configgen.jar takes the destination dirname as a property parameter set(CONFIG_DEST_DIR ${CMAKE_CURRENT_BINARY_DIR}) - # Get parent of destination directory - set(CONFIG_DEST_PARENT_DIR ${CONFIG_DEST_DIR}/..) - # Get destination dirname get_filename_component(CONFIG_DEST_DIRNAME ${CMAKE_CURRENT_BINARY_DIR} NAME) @@ -144,8 +140,8 @@ function(vespa_generate_config TARGET RELATIVE_CONFIG_DEF_PATH) add_custom_command( OUTPUT ${CONFIG_H_PATH} ${CONFIG_CPP_PATH} - COMMAND java -Dconfig.spec=${CONFIG_DEF_PATH} -Dconfig.dest=${CONFIG_DEST_PARENT_DIR} -Dconfig.lang=cpp -Dconfig.subdir=${CONFIG_DEST_DIRNAME} -Dconfig.dumpTree=false -Xms64m -Xmx64m -jar ${PROJECT_SOURCE_DIR}/configgen/target/configgen.jar - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/.. + COMMAND java -Dconfig.spec=${CONFIG_DEF_PATH} -Dconfig.dest=${CONFIG_DEST_DIR} -Dconfig.lang=cpp -Dconfig.dumpTree=false -Xms64m -Xmx64m -jar ${PROJECT_SOURCE_DIR}/configgen/target/configgen.jar + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMENT "Generating cpp config for ${CONFIG_NAME} in ${CMAKE_CURRENT_SOURCE_DIR}" MAIN_DEPENDENCY ${CONFIG_DEF_PATH} ) @@ -157,11 +153,6 @@ function(vespa_generate_config TARGET RELATIVE_CONFIG_DEF_PATH) # Add generated to sources for target target_sources(${TARGET} PRIVATE ${CONFIG_H_PATH} ${CONFIG_CPP_PATH}) - # Needed to be able to do a #include <CONFIG_DEST_DIRNAME/config-<name>.h> for this target - # This is used within the generated config-<name>.cpp - # TODO: Should modify configgen to use #include <vespa/<modulename>/config-<name>.h> instead - target_include_directories(${TARGET} PRIVATE ${CONFIG_DEST_PARENT_DIR}) - # Needed to be able to do a #include <config-<name>.h> for this target # This is used within some unit tests target_include_directories(${TARGET} PRIVATE ${CONFIG_DEST_DIR}) diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java index 6a733bd8942..000b6cd4149 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.api; import com.yahoo.config.provision.ApplicationId; @@ -340,7 +340,8 @@ public abstract class ControllerHttpClient { Slime slime = new Slime(); Cursor rootObject = slime.setObject(); deployment.version().ifPresent(version -> rootObject.setString("vespaVersion", version)); - rootObject.setBool("deployDirectly", true); + if (deployment.isDryRun()) + rootObject.setString("dryRun", "true"); return toJson(slime); } diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/Deployment.java b/hosted-api/src/main/java/ai/vespa/hosted/api/Deployment.java index d012d27fbd8..77cc116b413 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/Deployment.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/Deployment.java @@ -1,9 +1,8 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.api; import java.nio.file.Path; import java.util.Optional; -import java.util.OptionalLong; /** * A deployment intended for hosted Vespa, containing an application package and some meta data. @@ -12,23 +11,31 @@ public class Deployment { private final Optional<String> version; private final Path applicationZip; + private final boolean dryRun; - private Deployment(Optional<String> version, Path applicationZip) { + private Deployment(Optional<String> version, Path applicationZip, boolean dryRun) { this.version = version; this.applicationZip = applicationZip; + this.dryRun = dryRun; } /** Returns a deployment which will use the provided application package. */ public static Deployment ofPackage(Path applicationZipFile) { - return new Deployment(Optional.empty(), applicationZipFile); + return new Deployment(Optional.empty(), applicationZipFile, false); } /** Returns a copy of this which will have the specified Vespa version on its nodes. */ public Deployment atVersion(String vespaVersion) { - return new Deployment(Optional.of(vespaVersion), applicationZip); + return new Deployment(Optional.of(vespaVersion), applicationZip, false); + } + + /** Returns a copy of this which will do a dry-run deployment. */ + public Deployment withDryRun() { + return new Deployment(version, applicationZip, true); } public Optional<String> version() { return version; } public Path applicationZip() { return applicationZip; } + public boolean isDryRun() { return dryRun; } } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EncodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EncodeExpression.java index 09034659ad0..f84da9ddef8 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EncodeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EncodeExpression.java @@ -27,6 +27,11 @@ public class EncodeExpression extends Expression { } @Override + public void setStatementOutputType(DataType type) { + targetType = ((TensorDataType)type).getTensorType(); + } + + @Override protected void doExecute(ExecutionContext context) { StringFieldValue input = (StringFieldValue) context.getValue(); Tensor tensor = encoder.encode(input.getString(), context.getLanguage(), targetType); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java index a121df8e5a8..67459c2b035 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java @@ -31,6 +31,8 @@ public abstract class Expression extends Selectable { this.inputType = inputType; } + public void setStatementOutputType(DataType type) {} + public final FieldValue execute(FieldValue val) { return execute(new ExecutionContext().setValue(val)); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java index 0ac195efb5d..0f7c2a411de 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java @@ -24,6 +24,12 @@ public abstract class ExpressionList<T extends Expression> extends CompositeExpr } } + @Override + public void setStatementOutputType(DataType type) { + for (Expression expression : expressions) + expression.setStatementOutputType(type); + } + public int size() { return expressions.size(); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java index 179f202788c..78c261caccb 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java @@ -27,6 +27,11 @@ public final class GuardExpression extends CompositeExpression { } @Override + public void setStatementOutputType(DataType type) { + exp.setStatementOutputType(type); + } + + @Override protected void doExecute(ExecutionContext context) { if (!shouldExecute && context.getAdapter() instanceof UpdateAdapter) { context.setValue(null); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OutputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OutputExpression.java index 398c2751bd8..267fb6fc51b 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OutputExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OutputExpression.java @@ -2,6 +2,11 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; +import com.yahoo.vespa.objects.ObjectOperation; +import com.yahoo.vespa.objects.ObjectPredicate; + +import java.util.ArrayList; +import java.util.List; /** * @author Simon Thoresen Hult @@ -58,4 +63,22 @@ public abstract class OutputExpression extends Expression { return getClass().hashCode() + (fieldName != null ? fieldName.hashCode() : 0); } + public static class OutputFieldNameExtractor implements ObjectOperation, ObjectPredicate { + + private final List<String> outputFieldNames = new ArrayList<>(1); + + public List<String> getOutputFieldNames() { return outputFieldNames; } + + @Override + public void execute(Object obj) { + outputFieldNames.add(((OutputExpression) obj).getFieldName()); + } + + @Override + public boolean check(Object obj) { + return obj instanceof OutputExpression; + } + + } + } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/AbstractResource.java b/jdisc_core/src/main/java/com/yahoo/jdisc/AbstractResource.java index b561f08af92..05c3582a541 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/AbstractResource.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/AbstractResource.java @@ -63,9 +63,8 @@ public abstract class AbstractResource implements SharedResource { activeReferences.add(referenceStack); state = currentStateDebugWithLock(); } - log.log(Level.WARNING, - getClass().getName() + "@" + System.identityHashCode(this) + ".refer(): state={ " + state + " }", - referenceStack); + log.log(Level.FINE, referenceStack, () -> + getClass().getName() + "@" + System.identityHashCode(this) + ".refer(): state={ " + state + " }"); return new DebugResourceReference(this, referenceStack); } @@ -87,9 +86,8 @@ public abstract class AbstractResource implements SharedResource { } doDestroy = activeReferences.isEmpty(); } - log.log(Level.WARNING, - getClass().getName() + "@" + System.identityHashCode(this) + " release: state={ " + state + " }", - releaseStack); + log.log(Level.FINE, releaseStack, + () ->getClass().getName() + "@" + System.identityHashCode(this) + " release: state={ " + state + " }"); if (doDestroy) { destroy(); } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ContainerWatchdog.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ContainerWatchdog.java index 30aa0028465..7e82955eede 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ContainerWatchdog.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ContainerWatchdog.java @@ -22,8 +22,9 @@ import java.util.logging.Logger; */ class ContainerWatchdog implements ContainerWatchdogMetrics, AutoCloseable { - static final Duration GRACE_PERIOD = Duration.ofMinutes(30); + static final Duration GRACE_PERIOD = Duration.ofMinutes(5); static final Duration UPDATE_PERIOD = Duration.ofMinutes(5); + static final Duration CHECK_PERIOD = Duration.ofSeconds(1); private static final Logger log = Logger.getLogger(ContainerWatchdog.class.getName()); @@ -35,6 +36,7 @@ class ContainerWatchdog implements ContainerWatchdogMetrics, AutoCloseable { private ActiveContainer currentContainer; private Instant currentContainerActivationTime; private int numStaleContainers; + private Instant lastLogTime; ContainerWatchdog() { this(new ScheduledThreadPoolExecutor( @@ -50,8 +52,8 @@ class ContainerWatchdog implements ContainerWatchdogMetrics, AutoCloseable { ContainerWatchdog(ScheduledExecutorService scheduler, Clock clock) { this.scheduler = scheduler; this.clock = clock; - scheduler.scheduleAtFixedRate( - this::monitorDeactivatedContainers, UPDATE_PERIOD.getSeconds(), UPDATE_PERIOD.getSeconds(), TimeUnit.SECONDS); + this.lastLogTime = clock.instant(); + scheduler.scheduleAtFixedRate(this::monitorDeactivatedContainers, CHECK_PERIOD.getSeconds(), CHECK_PERIOD.getSeconds(), TimeUnit.SECONDS); } @Override @@ -77,31 +79,43 @@ class ContainerWatchdog implements ContainerWatchdogMetrics, AutoCloseable { void onContainerActivation(ActiveContainer nextContainer) { synchronized (monitor) { if (currentContainer != null) { - deactivatedContainers.add( - new DeactivatedContainer(currentContainer, currentContainerActivationTime, clock.instant())); + deactivatedContainers.add(new DeactivatedContainer(currentContainer, currentContainerActivationTime, clock.instant())); } currentContainer = nextContainer; currentContainerActivationTime = clock.instant(); } } + private String removalMsg(DeactivatedContainer container) { + return String.format("Removing deactivated container: instance=%s, activated=%s, deactivated=%s", + container.instance, container.timeActivated, container.timeDeactivated); + } + + private String regularMsg(DeactivatedContainer container, int refCount) { + return String.format( + "Deactivated container still alive: instance=%s, activated=%s, deactivated=%s, ref-count=%d", + container.instance, container.timeActivated, container.timeDeactivated, refCount); + } + void monitorDeactivatedContainers() { synchronized (monitor) { int numStaleContainer = 0; Iterator<DeactivatedContainer> iterator = deactivatedContainers.iterator(); + boolean timeToLogAgain = clock.instant().isAfter(lastLogTime.plus(UPDATE_PERIOD)); while (iterator.hasNext()) { DeactivatedContainer container = iterator.next(); int refCount = container.instance.retainCount(); if (refCount == 0) { + log.fine(removalMsg(container)); iterator.remove(); - break; - } - if (isPastGracePeriod(container)) { - ++numStaleContainer; - log.warning( - String.format( - "Deactivated container still alive: instance=%s, activated=%s, deactivated=%s, ref-count=%d", - container.instance.toString(), container.timeActivated, container.timeDeactivated, refCount)); + } else { + if (isPastGracePeriod(container)) { + ++numStaleContainer; + if (timeToLogAgain) { + log.warning(regularMsg(container, refCount)); + lastLogTime = clock.instant(); + } + } } } this.numStaleContainers = numStaleContainer; diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/service/ClientProvider.java b/jdisc_core/src/main/java/com/yahoo/jdisc/service/ClientProvider.java index c178147a952..a47c6b06321 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/service/ClientProvider.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/service/ClientProvider.java @@ -2,7 +2,11 @@ package com.yahoo.jdisc.service; import com.yahoo.jdisc.Container; -import com.yahoo.jdisc.application.*; +import com.yahoo.jdisc.application.BindingRepository; +import com.yahoo.jdisc.application.ContainerActivator; +import com.yahoo.jdisc.application.ContainerBuilder; +import com.yahoo.jdisc.application.Application; +import com.yahoo.jdisc.application.UriPattern; import com.yahoo.jdisc.handler.RequestHandler; /** diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/service/ServerProvider.java b/jdisc_core/src/main/java/com/yahoo/jdisc/service/ServerProvider.java index b58f3bc5138..3b5cbfd9cbc 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/service/ServerProvider.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/service/ServerProvider.java @@ -37,7 +37,7 @@ public interface ServerProvider extends SharedResource { * {@link Container} does <em>not</em> call this method, instead it is a required step in the {@link Application} * initialization code.</p> */ - public void start(); + void start(); /** * <p>This is a synchronous method to close the listen port (or equivalent) of this ServerProvider and flush any @@ -48,5 +48,5 @@ public interface ServerProvider extends SharedResource { * <p>The {@link Container} does <em>not</em> call this method, instead it is a required step in the {@link * Application} shutdown code.</p> */ - public void close(); + void close(); } diff --git a/linguistics-components/.gitignore b/linguistics-components/.gitignore new file mode 100644 index 00000000000..8b990078588 --- /dev/null +++ b/linguistics-components/.gitignore @@ -0,0 +1,5 @@ +target +*.iml +*.ipr +*.iws +/pom.xml.build diff --git a/linguistics-components/CMakeLists.txt b/linguistics-components/CMakeLists.txt new file mode 100644 index 00000000000..b53c8001959 --- /dev/null +++ b/linguistics-components/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +install_fat_java_artifact(linguistics-components) + +install_config_definitions() diff --git a/linguistics-components/OWNERS b/linguistics-components/OWNERS new file mode 100644 index 00000000000..cd50f7a263a --- /dev/null +++ b/linguistics-components/OWNERS @@ -0,0 +1,2 @@ +bratseth +arnej27959 diff --git a/linguistics-components/README b/linguistics-components/README new file mode 100644 index 00000000000..e26a51e2f53 --- /dev/null +++ b/linguistics-components/README @@ -0,0 +1,4 @@ +Java library for linguistic operations in Vespa. + +This API is pluggable - multiple implementations may be supplied. +This module contains a default pure Java implementation, "simple".
\ No newline at end of file diff --git a/linguistics-components/abi-spec.json b/linguistics-components/abi-spec.json new file mode 100644 index 00000000000..5b6729c58ef --- /dev/null +++ b/linguistics-components/abi-spec.json @@ -0,0 +1,189 @@ +{ + "com.yahoo.language.sentencepiece.Scoring": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static com.yahoo.language.sentencepiece.Scoring[] values()", + "public static com.yahoo.language.sentencepiece.Scoring valueOf(java.lang.String)" + ], + "fields": [ + "public static final enum com.yahoo.language.sentencepiece.Scoring highestScore", + "public static final enum com.yahoo.language.sentencepiece.Scoring fewestSegments" + ] + }, + "com.yahoo.language.sentencepiece.SentencePieceConfig$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigInstance$Builder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig)", + "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder collapseUnknowns(boolean)", + "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder scoring(com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum)", + "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder model(com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder)", + "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder model(java.util.List)", + "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", + "public final java.lang.String getDefMd5()", + "public final java.lang.String getDefName()", + "public final java.lang.String getDefNamespace()", + "public final boolean getApplyOnRestart()", + "public final void setApplyOnRestart(boolean)", + "public com.yahoo.language.sentencepiece.SentencePieceConfig build()" + ], + "fields": [ + "public java.util.List model" + ] + }, + "com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigBuilder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Model)", + "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder language(java.lang.String)", + "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder path(com.yahoo.config.FileReference)", + "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model build()" + ], + "fields": [] + }, + "com.yahoo.language.sentencepiece.SentencePieceConfig$Model": { + "superClass": "com.yahoo.config.InnerNode", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder)", + "public java.lang.String language()", + "public java.nio.file.Path path()" + ], + "fields": [] + }, + "com.yahoo.language.sentencepiece.SentencePieceConfig$Producer": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigInstance$Producer" + ], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract void getConfig(com.yahoo.language.sentencepiece.SentencePieceConfig$Builder)" + ], + "fields": [] + }, + "com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum[] values()", + "public static com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum valueOf(java.lang.String)" + ], + "fields": [ + "public static final enum com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum highestScore", + "public static final enum com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum fewestSegments" + ] + }, + "com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring": { + "superClass": "com.yahoo.config.EnumNode", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum)" + ], + "fields": [ + "public static final com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum highestScore", + "public static final com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum fewestSegments" + ] + }, + "com.yahoo.language.sentencepiece.SentencePieceConfig": { + "superClass": "com.yahoo.config.ConfigInstance", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public static java.lang.String getDefMd5()", + "public static java.lang.String getDefName()", + "public static java.lang.String getDefNamespace()", + "public static java.lang.String getDefVersion()", + "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Builder)", + "public boolean collapseUnknowns()", + "public com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum scoring()", + "public java.util.List model()", + "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model model(int)" + ], + "fields": [ + "public static final java.lang.String CONFIG_DEF_MD5", + "public static final java.lang.String CONFIG_DEF_NAME", + "public static final java.lang.String CONFIG_DEF_NAMESPACE", + "public static final java.lang.String CONFIG_DEF_VERSION", + "public static final java.lang.String[] CONFIG_DEF_SCHEMA" + ] + }, + "com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void addModel(com.yahoo.language.Language, java.nio.file.Path)", + "public com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder addDefaultModel(java.nio.file.Path)", + "public java.util.Map getModels()", + "public com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder setCollapseUnknowns(boolean)", + "public boolean getCollapseUnknowns()", + "public com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder setScoring(com.yahoo.language.sentencepiece.Scoring)", + "public com.yahoo.language.sentencepiece.Scoring getScoring()", + "public com.yahoo.language.sentencepiece.SentencePieceEncoder build()" + ], + "fields": [] + }, + "com.yahoo.language.sentencepiece.SentencePieceEncoder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.language.process.Segmenter", + "com.yahoo.language.process.Encoder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig)", + "public void <init>(com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder)", + "public java.util.List segment(java.lang.String, com.yahoo.language.Language)", + "public java.util.List encode(java.lang.String, com.yahoo.language.Language)", + "public com.yahoo.tensor.Tensor encode(java.lang.String, com.yahoo.language.Language, com.yahoo.tensor.TensorType)", + "public java.lang.String normalize(java.lang.String)" + ], + "fields": [] + } +}
\ No newline at end of file diff --git a/linguistics-components/pom.xml b/linguistics-components/pom.xml new file mode 100644 index 00000000000..44e58fb7588 --- /dev/null +++ b/linguistics-components/pom.xml @@ -0,0 +1,80 @@ +<?xml version="1.0"?> +<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>linguistics-components</artifactId> + <packaging>container-plugin</packaging> + <version>7-SNAPSHOT</version> + <dependencies> + <dependency> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>component</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>linguistics</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.google.inject</groupId> + <artifactId>guice</artifactId> + <scope>provided</scope> + <classifier>no_aop</classifier> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>com.github.os72</groupId> + <artifactId>protoc-jar-maven-plugin</artifactId> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <compilerArgs> + <arg>-Xlint:rawtypes</arg> + <arg>-Xlint:unchecked</arg> + <arg>-Xlint:deprecation</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>abi-check-plugin</artifactId> + </plugin> + </plugins> + </build> +</project> diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Model.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Model.java index 74f300057dc..74f300057dc 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Model.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Model.java diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/ResultBuilder.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/ResultBuilder.java index 2141505374c..2141505374c 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/ResultBuilder.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/ResultBuilder.java diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Scoring.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Scoring.java index 6c8560abee7..6c8560abee7 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Scoring.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Scoring.java diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java index 1659e3c0fa7..1659e3c0fa7 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java index b6659ebeaa3..b6659ebeaa3 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/TokenType.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/TokenType.java index 782030a8e4d..782030a8e4d 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/TokenType.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/TokenType.java diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Trie.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Trie.java index 8e7c2db2ed3..8e7c2db2ed3 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Trie.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/Trie.java diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/package-info.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/package-info.java index 4a8673705ec..3f97277c489 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/package-info.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/package-info.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2021 Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage @PublicApi package com.yahoo.language.sentencepiece; diff --git a/linguistics/src/main/protobuf/sentencepiece_model.proto b/linguistics-components/src/main/protobuf/sentencepiece_model.proto index 39626aede53..39626aede53 100644 --- a/linguistics/src/main/protobuf/sentencepiece_model.proto +++ b/linguistics-components/src/main/protobuf/sentencepiece_model.proto diff --git a/linguistics/src/main/resources/configdefinitions/sentence-piece.def b/linguistics-components/src/main/resources/configdefinitions/language.sentencepiece.sentence-piece.def index b91c0c45dc4..b91c0c45dc4 100644 --- a/linguistics/src/main/resources/configdefinitions/sentence-piece.def +++ b/linguistics-components/src/main/resources/configdefinitions/language.sentencepiece.sentence-piece.def diff --git a/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceConfigurationTest.java b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceConfigurationTest.java index edbbe21ec53..edbbe21ec53 100644 --- a/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceConfigurationTest.java +++ b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceConfigurationTest.java diff --git a/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java index d60d7386d4b..d60d7386d4b 100644 --- a/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java +++ b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java diff --git a/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTester.java b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTester.java index 1ba7c9b472d..1ba7c9b472d 100644 --- a/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTester.java +++ b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTester.java diff --git a/linguistics/src/test/models/sentencepiece/en.wiki.bpe.vs10000.model b/linguistics-components/src/test/models/sentencepiece/en.wiki.bpe.vs10000.model Binary files differindex 89f93ef3517..89f93ef3517 100644 --- a/linguistics/src/test/models/sentencepiece/en.wiki.bpe.vs10000.model +++ b/linguistics-components/src/test/models/sentencepiece/en.wiki.bpe.vs10000.model diff --git a/linguistics/src/test/models/sentencepiece/ja.wiki.bpe.vs5000.model b/linguistics-components/src/test/models/sentencepiece/ja.wiki.bpe.vs5000.model Binary files differindex 41c0688d9df..41c0688d9df 100644 --- a/linguistics/src/test/models/sentencepiece/ja.wiki.bpe.vs5000.model +++ b/linguistics-components/src/test/models/sentencepiece/ja.wiki.bpe.vs5000.model diff --git a/linguistics/abi-spec.json b/linguistics/abi-spec.json index dbf4842ea1a..cfbf2abda1a 100644 --- a/linguistics/abi-spec.json +++ b/linguistics/abi-spec.json @@ -731,192 +731,5 @@ "public abstract java.lang.String accentDrop(java.lang.String, com.yahoo.language.Language)" ], "fields": [] - }, - "com.yahoo.language.sentencepiece.Scoring": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.language.sentencepiece.Scoring[] values()", - "public static com.yahoo.language.sentencepiece.Scoring valueOf(java.lang.String)" - ], - "fields": [ - "public static final enum com.yahoo.language.sentencepiece.Scoring highestScore", - "public static final enum com.yahoo.language.sentencepiece.Scoring fewestSegments" - ] - }, - "com.yahoo.language.sentencepiece.SentencePieceConfig$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigInstance$Builder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder collapseUnknowns(boolean)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder scoring(com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder model(com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder model(java.util.List)", - "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", - "public final java.lang.String getDefMd5()", - "public final java.lang.String getDefName()", - "public final java.lang.String getDefNamespace()", - "public final boolean getApplyOnRestart()", - "public final void setApplyOnRestart(boolean)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig build()" - ], - "fields": [ - "public java.util.List model" - ] - }, - "com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigBuilder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Model)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder language(java.lang.String)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder path(com.yahoo.config.FileReference)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model build()" - ], - "fields": [] - }, - "com.yahoo.language.sentencepiece.SentencePieceConfig$Model": { - "superClass": "com.yahoo.config.InnerNode", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder)", - "public java.lang.String language()", - "public java.nio.file.Path path()" - ], - "fields": [] - }, - "com.yahoo.language.sentencepiece.SentencePieceConfig$Producer": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigInstance$Producer" - ], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void getConfig(com.yahoo.language.sentencepiece.SentencePieceConfig$Builder)" - ], - "fields": [] - }, - "com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum[] values()", - "public static com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum valueOf(java.lang.String)" - ], - "fields": [ - "public static final enum com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum highestScore", - "public static final enum com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum fewestSegments" - ] - }, - "com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring": { - "superClass": "com.yahoo.config.EnumNode", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum)" - ], - "fields": [ - "public static final com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum highestScore", - "public static final com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum fewestSegments" - ] - }, - "com.yahoo.language.sentencepiece.SentencePieceConfig": { - "superClass": "com.yahoo.config.ConfigInstance", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public static java.lang.String getDefMd5()", - "public static java.lang.String getDefName()", - "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", - "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Builder)", - "public boolean collapseUnknowns()", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum scoring()", - "public java.util.List model()", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model model(int)" - ], - "fields": [ - "public static final java.lang.String CONFIG_DEF_MD5", - "public static final java.lang.String CONFIG_DEF_NAME", - "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", - "public static final java.lang.String[] CONFIG_DEF_SCHEMA" - ] - }, - "com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void addModel(com.yahoo.language.Language, java.nio.file.Path)", - "public com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder addDefaultModel(java.nio.file.Path)", - "public java.util.Map getModels()", - "public com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder setCollapseUnknowns(boolean)", - "public boolean getCollapseUnknowns()", - "public com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder setScoring(com.yahoo.language.sentencepiece.Scoring)", - "public com.yahoo.language.sentencepiece.Scoring getScoring()", - "public com.yahoo.language.sentencepiece.SentencePieceEncoder build()" - ], - "fields": [] - }, - "com.yahoo.language.sentencepiece.SentencePieceEncoder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.language.process.Segmenter", - "com.yahoo.language.process.Encoder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig)", - "public void <init>(com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder)", - "public java.util.List segment(java.lang.String, com.yahoo.language.Language)", - "public java.util.List encode(java.lang.String, com.yahoo.language.Language)", - "public com.yahoo.tensor.Tensor encode(java.lang.String, com.yahoo.language.Language, com.yahoo.tensor.TensorType)", - "public java.lang.String normalize(java.lang.String)" - ], - "fields": [] } }
\ No newline at end of file diff --git a/linguistics/pom.xml b/linguistics/pom.xml index 221d7181616..0e5f9e15b85 100644 --- a/linguistics/pom.xml +++ b/linguistics/pom.xml @@ -15,10 +15,6 @@ <version>7-SNAPSHOT</version> <dependencies> <dependency> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - </dependency> - <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> diff --git a/logd/src/logd/log_protocol_proto.h b/logd/src/logd/log_protocol_proto.h index 6f82b5d28b8..2fcb7d03919 100644 --- a/logd/src/logd/log_protocol_proto.h +++ b/logd/src/logd/log_protocol_proto.h @@ -7,7 +7,7 @@ #pragma GCC diagnostic ignored "-Wsuggest-override" #endif -#include "log_protocol.pb.h" +#include <logd/log_protocol.pb.h> #pragma GCC diagnostic pop diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/NetworkMultiplexer.java b/messagebus/src/main/java/com/yahoo/messagebus/network/NetworkMultiplexer.java index e75b5d0934a..4bd9bbf7f7f 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/NetworkMultiplexer.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/NetworkMultiplexer.java @@ -7,10 +7,9 @@ import com.yahoo.text.Utf8Array; import java.util.Deque; import java.util.Map; -import java.util.Queue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; /** @@ -33,15 +32,16 @@ public class NetworkMultiplexer implements NetworkOwner { private final Network net; private final Deque<NetworkOwner> owners = new ConcurrentLinkedDeque<>(); private final Map<String, Deque<NetworkOwner>> sessions = new ConcurrentHashMap<>(); - private final boolean shared; + private final AtomicBoolean disowned; private NetworkMultiplexer(Network net, boolean shared) { net.attach(this); this.net = net; - this.shared = shared; + this.disowned = new AtomicBoolean( ! shared); } - /** Returns a network multiplexer which will be shared between several {@link NetworkOwner}s. */ + /** Returns a network multiplexer which will be shared between several {@link NetworkOwner}s, + * and will shut down when all these have detached, and {@link #disown()} has been called, in any order. */ public static NetworkMultiplexer shared(Network net) { return new NetworkMultiplexer(net, true); } @@ -100,6 +100,7 @@ public class NetworkMultiplexer implements NetworkOwner { owner.deliverMessage(message, session); } + /** Attach the network owner to this, allowing this to forward messages to it. */ public void attach(NetworkOwner owner) { if (owners.contains(owner)) throw new IllegalArgumentException(owner + " is already attached to this"); @@ -107,23 +108,27 @@ public class NetworkMultiplexer implements NetworkOwner { owners.add(owner); } + /** Detach the network owner from this, no longer allowing messages to it, and shutting down this is ownerless. */ public void detach(NetworkOwner owner) { if ( ! owners.remove(owner)) throw new IllegalArgumentException(owner + " not attached to this"); - if ( ! shared && owners.isEmpty()) - net.shutdown(); + destroyIfOwnerless(); } - public void destroy() { - if ( ! shared) - throw new UnsupportedOperationException("Destroy called on a dedicated multiplexer; " + - "this automatically shuts down when detached from"); + /** Signal that external ownership of this is relinquished, allowing destruction on last owner detachment. */ + public void disown() { + if (disowned.getAndSet(true)) + throw new IllegalStateException("Destroy called on a dedicated multiplexer--" + + "this automatically shuts down when detached from--or " + + "called multiple times on a shared multiplexer"); - if ( ! owners.isEmpty()) - log.warning("NetworkMultiplexer destroyed before all owners detached: " + this); + destroyIfOwnerless(); + } - net.shutdown(); + private void destroyIfOwnerless() { + if (disowned.get() && owners.isEmpty()) + net.shutdown(); } public Network net() { @@ -136,7 +141,7 @@ public class NetworkMultiplexer implements NetworkOwner { "net=" + net + ", owners=" + owners + ", sessions=" + sessions + - ", shared=" + shared + + ", destructible=" + disowned + '}'; } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/NamedRPCService.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/NamedRPCService.java new file mode 100644 index 00000000000..59cafed1836 --- /dev/null +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/NamedRPCService.java @@ -0,0 +1,48 @@ +package com.yahoo.messagebus.network.rpc; + +import com.yahoo.jrt.slobrok.api.IMirror; +import com.yahoo.jrt.slobrok.api.Mirror; + +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; + +public class NamedRPCService implements RPCService { + private final IMirror mirror; + private final String pattern; + private int addressIdx = ThreadLocalRandom.current().nextInt(Integer.MAX_VALUE); + private int addressGen = 0; + private List<Mirror.Entry> addressList = null; + + /** + * Create a new RPCService backed by the given network and using the given service pattern. + * + * @param mirror The naming server to send queries to. + * @param pattern The pattern to use when querying. + */ + public NamedRPCService(IMirror mirror, String pattern) { + this.mirror = mirror; + this.pattern = pattern; + } + + /** + * Resolve a concrete address from this service. This service may represent multiple remote sessions, so this will + * select one that is online. + * + * @return A concrete service address. + */ + public synchronized RPCServiceAddress resolve() { + if (addressGen != mirror.updates()) { + addressGen = mirror.updates(); + addressList = mirror.lookup(pattern); + } + if (addressList != null && !addressList.isEmpty()) { + ++addressIdx; + if (addressIdx >= addressList.size()) { + addressIdx = 0; + } + Mirror.Entry entry = addressList.get(addressIdx); + return new RPCServiceAddress(entry.getName(), entry.getSpec()); + } + return null; + } +} diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCService.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCService.java index fb3c4cf9971..889df32ce1e 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCService.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCService.java @@ -2,10 +2,6 @@ package com.yahoo.messagebus.network.rpc; import com.yahoo.jrt.slobrok.api.IMirror; -import com.yahoo.jrt.slobrok.api.Mirror; - -import java.util.List; -import java.util.concurrent.ThreadLocalRandom; /** * An RPCService represents a set of remote sessions matching a service pattern. The sessions are monitored using the @@ -13,23 +9,13 @@ import java.util.concurrent.ThreadLocalRandom; * * @author havardpe */ -public class RPCService { - - private final IMirror mirror; - private final String pattern; - private int addressIdx = ThreadLocalRandom.current().nextInt(Integer.MAX_VALUE); - private int addressGen = 0; - private List<Mirror.Entry> addressList = null; +public interface RPCService { - /** - * Create a new RPCService backed by the given network and using the given service pattern. - * - * @param mirror The naming server to send queries to. - * @param pattern The pattern to use when querying. - */ - public RPCService(IMirror mirror, String pattern) { - this.mirror = mirror; - this.pattern = pattern; + static RPCService create(IMirror mirror, String pattern) { + if (pattern.startsWith("tcp/")) { + return new TcpRPCService(pattern); + } + return new NamedRPCService(mirror, pattern); } /** @@ -38,38 +24,6 @@ public class RPCService { * * @return A concrete service address. */ - public RPCServiceAddress resolve() { - if (pattern.startsWith("tcp/")) { - int pos = pattern.lastIndexOf('/'); - if (pos > 0 && pos < pattern.length() - 1) { - RPCServiceAddress ret = new RPCServiceAddress(pattern, pattern.substring(0, pos)); - if (!ret.isMalformed()) { - return ret; - } - } - } else { - if (addressGen != mirror.updates()) { - addressGen = mirror.updates(); - addressList = mirror.lookup(pattern); - } - if (addressList != null && !addressList.isEmpty()) { - ++addressIdx; - if (addressIdx >= addressList.size()) { - addressIdx = 0; - } - Mirror.Entry entry = addressList.get(addressIdx); - return new RPCServiceAddress(entry.getName(), entry.getSpec()); - } - } - return null; - } + RPCServiceAddress resolve(); - /** - * Returns the pattern used when querying for the naming server for addresses. This is given at construtor time. - * - * @return The service pattern. - */ - String getPattern() { - return pattern; - } } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCServiceAddress.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCServiceAddress.java index 0a6a58d4e89..1b7bcf01731 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCServiceAddress.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCServiceAddress.java @@ -34,6 +34,13 @@ public class RPCServiceAddress implements ServiceAddress { public RPCServiceAddress(String serviceName, String connectionSpec) { this(serviceName, new Spec(connectionSpec)); } + public RPCServiceAddress(RPCServiceAddress blueprint) { + serviceName = blueprint.serviceName; + sessionName = blueprint.sessionName; + connectionSpec = blueprint.connectionSpec; + target = null; + } + @Override public boolean equals(Object obj) { diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCServicePool.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCServicePool.java index abd33d6c9c2..a666a03c401 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCServicePool.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCServicePool.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.messagebus.network.rpc; +import com.yahoo.concurrent.CopyOnWriteHashMap; + import java.util.LinkedHashMap; import java.util.Map; @@ -12,7 +14,7 @@ import java.util.Map; public class RPCServicePool { private final RPCNetwork net; - private final ThreadLocalCache services = new ThreadLocalCache(); + private final Map<Long, ServiceLRUCache> mapOfServiceCache; private final int maxSize; /** @@ -23,6 +25,7 @@ public class RPCServicePool { */ public RPCServicePool(RPCNetwork net, int maxSize) { this.net = net; + mapOfServiceCache = new CopyOnWriteHashMap<>(); this.maxSize = maxSize; } @@ -34,12 +37,12 @@ public class RPCServicePool { * @return A service address for the given pattern. */ public RPCServiceAddress resolve(String pattern) { - RPCService service = services.get().get(pattern); - if (service == null) { - service = new RPCService(net.getMirror(), pattern); - services.get().put(pattern, service); - } - return service.resolve(); + + return getPerThreadCache().computeIfAbsent(pattern, (key) -> RPCService.create(net.getMirror(), key)).resolve(); + } + + private ServiceLRUCache getPerThreadCache() { + return mapOfServiceCache.computeIfAbsent(Thread.currentThread().getId(), (key) -> new ServiceLRUCache(maxSize)); } /** @@ -49,7 +52,7 @@ public class RPCServicePool { * @return The current size of this pool. */ public int getSize() { - return services.get().size(); + return getPerThreadCache().size(); } /** @@ -59,21 +62,15 @@ public class RPCServicePool { * @return True if a corresponding service is in the pool. */ public boolean hasService(String pattern) { - return services.get().containsKey(pattern); - } - - private class ThreadLocalCache extends ThreadLocal<ServiceLRUCache> { - - @Override - protected ServiceLRUCache initialValue() { - return new ServiceLRUCache(); - } + return getPerThreadCache().containsKey(pattern); } - private class ServiceLRUCache extends LinkedHashMap<String, RPCService> { + private static class ServiceLRUCache extends LinkedHashMap<String, RPCService> { + private final int maxSize; - ServiceLRUCache() { + ServiceLRUCache(int maxSize) { super(16, 0.75f, true); + this.maxSize = maxSize; } @Override diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/TcpRPCService.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/TcpRPCService.java new file mode 100644 index 00000000000..e2fae59b429 --- /dev/null +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/TcpRPCService.java @@ -0,0 +1,23 @@ +package com.yahoo.messagebus.network.rpc; + +public class TcpRPCService implements RPCService { + private final RPCServiceAddress blueprint; + + TcpRPCService(String pattern) { + if ( ! pattern.startsWith("tcp/")) { + throw new IllegalArgumentException("Expect tcp adress to start with 'tcp/', was: " + pattern); + } + RPCServiceAddress ret = null; + int pos = pattern.lastIndexOf('/'); + if (pos > 0 && pos < pattern.length() - 1) { + ret = new RPCServiceAddress(pattern, pattern.substring(0, pos)); + if ( ret.isMalformed()) { + ret = null; + } + } + blueprint = ret; + } + public RPCServiceAddress resolve() { + return blueprint != null ? new RPCServiceAddress(blueprint) : null; + } +} diff --git a/messagebus/src/test/java/com/yahoo/messagebus/network/NetworkMultiplexerTest.java b/messagebus/src/test/java/com/yahoo/messagebus/network/NetworkMultiplexerTest.java index 3bf754f29c7..6451a692fb6 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/network/NetworkMultiplexerTest.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/network/NetworkMultiplexerTest.java @@ -86,7 +86,11 @@ public class NetworkMultiplexerTest { shared.detach(owner2); assertFalse(net.shutDown.get()); - shared.destroy(); + shared.attach(owner2); + shared.disown(); + assertFalse(net.shutDown.get()); + + shared.detach(owner2); assertTrue(net.shutDown.get()); } diff --git a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/ServiceAddressTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/ServiceAddressTestCase.java index 1dbb30de585..0343b075579 100755 --- a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/ServiceAddressTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/ServiceAddressTestCase.java @@ -83,11 +83,11 @@ public class ServiceAddressTestCase { } private void assertNullAddress(String pattern) { - assertNull(new RPCService(network.getMirror(), pattern).resolve()); + assertNull(RPCService.create(network.getMirror(), pattern).resolve()); } private void assertAddress(String pattern, String expectedSpec, String expectedSession) { - RPCService service = new RPCService(network.getMirror(), pattern); + RPCService service = RPCService.create(network.getMirror(), pattern); RPCServiceAddress obj = service.resolve(); assertNotNull(obj); assertNotNull(obj.getConnectionSpec()); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java index b4c116d1903..117503b95dc 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java @@ -32,7 +32,7 @@ import java.util.logging.Logger; */ public abstract class HttpMetricFetcher { - private final static Logger log = Logger.getLogger(HttpMetricFetcher.class.getPackage().getName()); + private final static Logger log = Logger.getLogger(HttpMetricFetcher.class.getName()); public final static String STATE_PATH = "/state/v1/"; // The call to apache will do 3 retries. As long as we check the services in series, we can't have this too high. public static int CONNECTION_TIMEOUT = 5000; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java index 59db14670aa..03e72ec36de 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java @@ -5,7 +5,6 @@ import ai.vespa.metricsproxy.metric.HealthMetric; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.ByteArrayInputStream; import java.io.InputStream; import java.util.concurrent.ExecutionException; import java.util.logging.Level; @@ -36,19 +35,19 @@ public class RemoteHealthMetricFetcher extends HttpMetricFetcher { return createHealthMetrics(stream, fetchCount); } catch (IOException | InterruptedException | ExecutionException e) { logMessageNoResponse(errMsgNoResponse(e), fetchCount); - byte [] empty = {'{','}'}; - return createHealthMetrics(new ByteArrayInputStream(empty), fetchCount); + return HealthMetric.getUnknown("Failed fetching metrics for service: " + service.getMonitoringName()); } } /** * Connect to remote service over http and fetch metrics */ - private HealthMetric createHealthMetrics(InputStream data, int fetchCount) { + private HealthMetric createHealthMetrics(InputStream data, int fetchCount) throws IOException { try { return parse(data); } catch (Exception e) { handleException(e, data, fetchCount); + while (data.read() != -1) {} return HealthMetric.getDown("Failed fetching status page for service"); } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java index 3ee1e05c263..aad2f816959 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java @@ -35,11 +35,12 @@ public class RemoteMetricsFetcher extends HttpMetricFetcher { handleException(e, data, fetchCount); } } - private void createMetrics(InputStream data, MetricsParser.Consumer consumer, int fetchCount) { + private void createMetrics(InputStream data, MetricsParser.Consumer consumer, int fetchCount) throws IOException { try { MetricsParser.parse(data, consumer); } catch (Exception e) { handleException(e, data, fetchCount); + while (data.read() != -1) {} } } } diff --git a/metrics/src/vespa/metrics/metricmanager.h b/metrics/src/vespa/metrics/metricmanager.h index f5ad5c5eea3..15f81126669 100644 --- a/metrics/src/vespa/metrics/metricmanager.h +++ b/metrics/src/vespa/metrics/metricmanager.h @@ -43,7 +43,7 @@ */ #pragma once -#include "config-metricsmanager.h" +#include <vespa/metrics/config-metricsmanager.h> #include "metricset.h" #include "metricsnapshot.h" #include "memoryconsumption.h" diff --git a/model-integration/pom.xml b/model-integration/pom.xml index dc3154c5c41..62014ef174a 100644 --- a/model-integration/pom.xml +++ b/model-integration/pom.xml @@ -106,29 +106,4 @@ </plugins> </build> - <profiles> - <!-- Exclude TF JNI when building for rhel6, which needs a special, natively installed variant --> - <profile> - <id>rhel6</id> - <activation> - <property> - <name>target.env</name> - <value>rhel6</value> - </property> - </activation> - <dependencies> - <dependency> - <groupId>org.tensorflow</groupId> - <artifactId>tensorflow</artifactId> - <exclusions> - <exclusion> - <groupId>org.tensorflow</groupId> - <artifactId>libtensorflow_jni</artifactId> - </exclusion> - </exclusions> - </dependency> - </dependencies> - </profile> - </profiles> - </project> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java index 2379659f74b..32336743bdc 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java @@ -6,9 +6,11 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.DockerImage; import java.time.Instant; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -29,6 +31,7 @@ public class NodeAttributes { private Optional<Version> vespaVersion = Optional.empty(); private Optional<Version> currentOsVersion = Optional.empty(); private Optional<Instant> currentFirmwareCheck = Optional.empty(); + private List<TrustStoreItem> trustStore = List.of(); /** The list of reports to patch. A null value is used to remove the report. */ private Map<String, JsonNode> reports = new TreeMap<>(); @@ -144,7 +147,17 @@ public class NodeAttributes { && Objects.equals(vespaVersion, other.vespaVersion) && Objects.equals(currentOsVersion, other.currentOsVersion) && Objects.equals(currentFirmwareCheck, other.currentFirmwareCheck) - && Objects.equals(reports, other.reports); + && Objects.equals(reports, other.reports) + && Objects.equals(trustStore, other.trustStore); + } + + public NodeAttributes withTrustStore(List<TrustStoreItem> trustStore) { + this.trustStore = List.copyOf(trustStore); + return this; + } + + public List<TrustStoreItem> getTrustStore() { + return trustStore; } @Override @@ -156,7 +169,8 @@ public class NodeAttributes { vespaVersion.map(ver -> "vespaVersion=" + ver.toFullString()), currentOsVersion.map(ver -> "currentOsVersion=" + ver.toFullString()), currentFirmwareCheck.map(at -> "currentFirmwareCheck=" + at), - Optional.ofNullable(reports.isEmpty() ? null : "reports=" + reports)) + Optional.ofNullable(reports.isEmpty() ? null : "reports=" + reports), + Optional.ofNullable(trustStore.isEmpty() ? null : "trustStore=" + trustStore)) .filter(Optional::isPresent) .map(Optional::get) .collect(Collectors.joining(", ", "{", "}")); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java index e85d51ef992..4000a0ac182 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java @@ -68,6 +68,8 @@ public class NodeSpec { private final Optional<ApplicationId> exclusiveTo; + private final List<TrustStoreItem> trustStore; + public NodeSpec( String hostname, Optional<String> id, @@ -98,7 +100,8 @@ public class NodeSpec { List<Event> events, Optional<String> parentHostname, Optional<URI> archiveUri, - Optional<ApplicationId> exclusiveTo) { + Optional<ApplicationId> exclusiveTo, + List<TrustStoreItem> trustStore) { if (state == NodeState.active) { requireOptional(owner, "owner"); requireOptional(membership, "membership"); @@ -138,6 +141,7 @@ public class NodeSpec { this.parentHostname = Objects.requireNonNull(parentHostname); this.archiveUri = Objects.requireNonNull(archiveUri); this.exclusiveTo = Objects.requireNonNull(exclusiveTo); + this.trustStore = Objects.requireNonNull(trustStore); } public String hostname() { @@ -283,6 +287,10 @@ public class NodeSpec { return exclusiveTo; } + public List<TrustStoreItem> trustStore() { + return trustStore; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -319,7 +327,8 @@ public class NodeSpec { Objects.equals(events, that.events) && Objects.equals(parentHostname, that.parentHostname) && Objects.equals(archiveUri, that.archiveUri) && - Objects.equals(exclusiveTo, that.exclusiveTo); + Objects.equals(exclusiveTo, that.exclusiveTo) && + Objects.equals(trustStore, that.trustStore); } @Override @@ -354,7 +363,8 @@ public class NodeSpec { events, parentHostname, archiveUri, - exclusiveTo); + exclusiveTo, + trustStore); } @Override @@ -390,6 +400,7 @@ public class NodeSpec { + " parentHostname=" + parentHostname + " archiveUri=" + archiveUri + " exclusiveTo=" + exclusiveTo + + " trustStore=" + trustStore + " }"; } @@ -424,6 +435,7 @@ public class NodeSpec { private Optional<String> parentHostname = Optional.empty(); private Optional<URI> archiveUri = Optional.empty(); private Optional<ApplicationId> exclusiveTo = Optional.empty(); + private List<TrustStoreItem> trustStore = List.of(); public Builder() {} @@ -456,6 +468,7 @@ public class NodeSpec { node.parentHostname.ifPresent(this::parentHostname); node.archiveUri.ifPresent(this::archiveUri); node.exclusiveTo.ifPresent(this::exclusiveTo); + trustStore(node.trustStore); } public Builder hostname(String hostname) { @@ -633,12 +646,19 @@ public class NodeSpec { return this; } + public Builder trustStore(List<TrustStoreItem> trustStore) { + this.trustStore = List.copyOf(trustStore); + return this; + } + public Builder updateFromNodeAttributes(NodeAttributes attributes) { attributes.getHostId().ifPresent(this::id); attributes.getDockerImage().ifPresent(this::currentDockerImage); attributes.getCurrentOsVersion().ifPresent(this::currentOsVersion); attributes.getRebootGeneration().ifPresent(this::currentRebootGeneration); attributes.getRestartGeneration().ifPresent(this::currentRestartGeneration); + // Always replace entire trust store + trustStore(attributes.getTrustStore()); this.reports.updateFromRawMap(attributes.getReports()); return this; @@ -752,7 +772,7 @@ public class NodeSpec { wantedRebootGeneration, currentRebootGeneration, wantedFirmwareCheck, currentFirmwareCheck, modelName, resources, realResources, ipAddresses, additionalIpAddresses, - reports, events, parentHostname, archiveUri, exclusiveTo); + reports, events, parentHostname, archiveUri, exclusiveTo, trustStore); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java index abc779d8a9a..d5e3acf0656 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java @@ -155,6 +155,11 @@ public class RealNodeRepository implements NodeRepository { .map(event -> new Event(event.agent, event.event, Optional.ofNullable(event.at).map(Instant::ofEpochMilli).orElse(Instant.EPOCH))) .collect(Collectors.toUnmodifiableList()); + List<TrustStoreItem> trustStore = Optional.ofNullable(node.trustStore).orElse(List.of()).stream() + .map(item -> new TrustStoreItem(item.fingerprint, Instant.ofEpochMilli(item.expiry))) + .collect(Collectors.toList()); + + return new NodeSpec( node.hostname, Optional.ofNullable(node.openStackId), @@ -185,7 +190,8 @@ public class RealNodeRepository implements NodeRepository { events, Optional.ofNullable(node.parentHostname), Optional.ofNullable(node.archiveUri).map(URI::create), - Optional.ofNullable(node.exclusiveTo).map(ApplicationId::fromSerializedForm)); + Optional.ofNullable(node.exclusiveTo).map(ApplicationId::fromSerializedForm), + trustStore); } private static NodeResources nodeResources(NodeRepositoryNode.NodeResources nodeResources) { @@ -270,7 +276,9 @@ public class RealNodeRepository implements NodeRepository { node.vespaVersion = nodeAttributes.getVespaVersion().map(Version::toFullString).orElse(null); node.currentOsVersion = nodeAttributes.getCurrentOsVersion().map(Version::toFullString).orElse(null); node.currentFirmwareCheck = nodeAttributes.getCurrentFirmwareCheck().map(Instant::toEpochMilli).orElse(null); - + node.trustStore = nodeAttributes.getTrustStore().stream() + .map(item -> new NodeRepositoryNode.TrustStoreItem(item.fingerprint(), item.expiry().toEpochMilli())) + .collect(Collectors.toList()); Map<String, JsonNode> reports = nodeAttributes.getReports(); node.reports = reports == null || reports.isEmpty() ? null : new TreeMap<>(reports); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/TrustStoreItem.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/TrustStoreItem.java new file mode 100644 index 00000000000..d3e797bca24 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/TrustStoreItem.java @@ -0,0 +1,48 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; + +import java.time.Instant; +import java.util.Objects; + +/** + * @author mortent + */ +public class TrustStoreItem { + private final String fingerprint; + private final Instant expiry; + + public TrustStoreItem(String fingerprint, Instant expiry) { + this.fingerprint = fingerprint; + this.expiry = expiry; + } + + public String fingerprint() { + return fingerprint; + } + + public Instant expiry() { + return expiry; + } + + @Override + public String toString() { + return "TrustStoreItem{" + + "fingerprint='" + fingerprint + '\'' + + ", expiry=" + expiry + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TrustStoreItem that = (TrustStoreItem) o; + return Objects.equals(fingerprint, that.fingerprint) && Objects.equals(expiry, that.expiry); + } + + @Override + public int hashCode() { + return Objects.hash(fingerprint, expiry); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java index 4282c67b4cd..f471f9a9965 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java @@ -85,6 +85,9 @@ public class NodeRepositoryNode { public String exclusiveTo; @JsonProperty("history") public List<Event> history; + @JsonProperty("trustStore") + @JsonInclude(JsonInclude.Include.NON_EMPTY) + public List<TrustStoreItem> trustStore; @JsonProperty("reports") public Map<String, JsonNode> reports = null; @@ -221,4 +224,17 @@ public class NodeRepositoryNode { '}'; } } + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(JsonInclude.Include.NON_NULL) + public static class TrustStoreItem { + @JsonProperty ("fingerprint") + public String fingerprint; + @JsonProperty ("expiry") + public long expiry; + + public TrustStoreItem(@JsonProperty("fingerprint") String fingerprint, @JsonProperty("expiry") long expiry) { + this.fingerprint = fingerprint; + this.expiry = expiry; + } + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 240e041a504..df3ac00ce7f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -18,13 +18,16 @@ import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.NodeAcl; import com.yahoo.vespa.hosted.provision.node.Reports; import com.yahoo.vespa.hosted.provision.node.Status; +import com.yahoo.vespa.hosted.provision.node.TrustStoreItem; import java.time.Instant; import java.util.Arrays; import java.util.EnumSet; +import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; /** * A node in the node repository. The identity of a node is given by its id. @@ -50,6 +53,7 @@ public final class Node implements Nodelike { private final Optional<ApplicationId> exclusiveToApplicationId; private final Optional<ClusterSpec.Type> exclusiveToClusterType; private final Optional<String> switchHostname; + private final List<TrustStoreItem> trustStoreItems; /** Record of the last event of each type happening to this node */ private final History history; @@ -79,7 +83,7 @@ public final class Node implements Nodelike { Flavor flavor, Status status, State state, Optional<Allocation> allocation, History history, NodeType type, Reports reports, Optional<String> modelName, Optional<TenantName> reservedTo, Optional<ApplicationId> exclusiveToApplicationId, Optional<ClusterSpec.Type> exclusiveToClusterType, - Optional<String> switchHostname) { + Optional<String> switchHostname, List<TrustStoreItem> trustStoreItems) { this.id = Objects.requireNonNull(id, "A node must have an ID"); this.hostname = requireNonEmptyString(hostname, "A node must have a hostname"); this.ipConfig = Objects.requireNonNull(ipConfig, "A node must a have an IP config"); @@ -96,6 +100,7 @@ public final class Node implements Nodelike { this.exclusiveToApplicationId = Objects.requireNonNull(exclusiveToApplicationId, "exclusiveToApplicationId cannot be null"); this.exclusiveToClusterType = Objects.requireNonNull(exclusiveToClusterType, "exclusiveToClusterType cannot be null"); this.switchHostname = requireNonEmptyString(switchHostname, "switchHostname cannot be null"); + this.trustStoreItems = trustStoreItems.stream().distinct().collect(Collectors.toUnmodifiableList()); if (state == State.active) requireNonEmpty(ipConfig.primary(), "Active node " + hostname + " must have at least one valid IP address"); @@ -207,6 +212,11 @@ public final class Node implements Nodelike { return switchHostname; } + /** Returns the trusted certificates for this host if any. */ + public List<TrustStoreItem> trustedCertificates() { + return trustStoreItems; + } + /** * Returns a copy of this where wantToFail is set to true and history is updated to reflect this. */ @@ -295,13 +305,13 @@ public final class Node implements Nodelike { /** Returns a node with the status assigned to the given value */ public Node with(Status status) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, - reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname); + reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); } /** Returns a node with the type assigned to the given value */ public Node with(NodeType type) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, - reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname); + reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); } /** Returns a node with the flavor assigned to the given value */ @@ -309,31 +319,31 @@ public final class Node implements Nodelike { if (flavor.equals(this.flavor)) return this; History updateHistory = history.with(new History.Event(History.Event.Type.resized, agent, instant)); return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, updateHistory, type, - reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname); + reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); } /** Returns a copy of this with the reboot generation set to generation */ public Node withReboot(Generation generation) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status.withReboot(generation), state, - allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname); + allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); } /** Returns a copy of this with the openStackId set */ public Node withOpenStackId(String openStackId) { return new Node(openStackId, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname); + allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); } /** Returns a copy of this with model name set to given value */ public Node withModelName(String modelName) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, Optional.of(modelName), reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname); + allocation, history, type, reports, Optional.of(modelName), reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); } /** Returns a copy of this with model name cleared */ public Node withoutModelName() { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, Optional.empty(), reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname); + allocation, history, type, reports, Optional.empty(), reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); } /** Returns a copy of this with a history record saying it was detected to be down at this instant */ @@ -364,55 +374,55 @@ public final class Node implements Nodelike { */ public Node with(Allocation allocation) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - Optional.of(allocation), history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname); + Optional.of(allocation), history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); } /** Returns a new Node without an allocation. */ public Node withoutAllocation() { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - Optional.empty(), history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname); + Optional.empty(), history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); } /** Returns a copy of this node with IP config set to the given value. */ public Node with(IP.Config ipConfig) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname); + allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); } /** Returns a copy of this node with the parent hostname assigned to the given value. */ public Node withParentHostname(String parentHostname) { return new Node(id, ipConfig, hostname, Optional.of(parentHostname), flavor, status, state, - allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname); + allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); } public Node withReservedTo(TenantName tenant) { if (type != NodeType.host) throw new IllegalArgumentException("Only host nodes can be reserved, " + hostname + " has type " + type); return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, Optional.of(tenant), exclusiveToApplicationId, exclusiveToClusterType, switchHostname); + allocation, history, type, reports, modelName, Optional.of(tenant), exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); } /** Returns a copy of this node which is not reserved to a tenant */ public Node withoutReservedTo() { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, Optional.empty(), exclusiveToApplicationId, exclusiveToClusterType, switchHostname); + allocation, history, type, reports, modelName, Optional.empty(), exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); } public Node withExclusiveToApplicationId(ApplicationId exclusiveTo) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo, Optional.ofNullable(exclusiveTo), exclusiveToClusterType, switchHostname); + allocation, history, type, reports, modelName, reservedTo, Optional.ofNullable(exclusiveTo), exclusiveToClusterType, switchHostname, trustStoreItems); } public Node withExclusiveToClusterType(ClusterSpec.Type exclusiveTo) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, Optional.ofNullable(exclusiveTo), switchHostname); + allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, Optional.ofNullable(exclusiveTo), switchHostname, trustStoreItems); } /** Returns a copy of this node with switch hostname set to given value */ public Node withSwitchHostname(String switchHostname) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, Optional.ofNullable(switchHostname)); + allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, Optional.ofNullable(switchHostname), trustStoreItems); } /** Returns a copy of this node with switch hostname unset */ @@ -450,12 +460,18 @@ public final class Node implements Nodelike { /** Returns a copy of this node with the given history. */ public Node with(History history) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname); + allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); } public Node with(Reports reports) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname); + allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); + } + + public Node with(List<TrustStoreItem> trustStoreItems) { + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, + allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, + trustStoreItems); } private static Optional<String> requireNonEmptyString(Optional<String> value, String message) { @@ -594,6 +610,7 @@ public final class Node implements Nodelike { private Status status; private Reports reports; private History history; + private List<TrustStoreItem> trustStoreItems; private Builder(String id, String hostname, Flavor flavor, State state, NodeType type) { this.id = id; @@ -663,12 +680,18 @@ public final class Node implements Nodelike { return this; } + public Builder trustedCertificates(List<TrustStoreItem> trustStoreItems) { + this.trustStoreItems = trustStoreItems; + return this; + } + public Node build() { return new Node(id, Optional.ofNullable(ipConfig).orElse(IP.Config.EMPTY), hostname, Optional.ofNullable(parentHostname), - flavor, Optional.ofNullable(status).orElseGet(Status::initial), state, Optional.ofNullable(allocation), - Optional.ofNullable(history).orElseGet(History::empty), type, Optional.ofNullable(reports).orElseGet(Reports::new), - Optional.ofNullable(modelName), Optional.ofNullable(reservedTo), Optional.ofNullable(exclusiveToApplicationId), - Optional.ofNullable(exclusiveToClusterType), Optional.ofNullable(switchHostname)); + flavor, Optional.ofNullable(status).orElseGet(Status::initial), state, Optional.ofNullable(allocation), + Optional.ofNullable(history).orElseGet(History::empty), type, Optional.ofNullable(reports).orElseGet(Reports::new), + Optional.ofNullable(modelName), Optional.ofNullable(reservedTo), Optional.ofNullable(exclusiveToApplicationId), + Optional.ofNullable(exclusiveToClusterType), Optional.ofNullable(switchHostname), + Optional.ofNullable(trustStoreItems).orElseGet(List::of)); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/TrustStoreItem.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/TrustStoreItem.java new file mode 100644 index 00000000000..6fb94d0bc62 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/TrustStoreItem.java @@ -0,0 +1,68 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.provision.node; + +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; + +import java.time.Instant; +import java.util.Objects; + +/** + * Contains the fingerprint and expiry of certificates in a hosts truststore. + * + * @author mortent + */ +public class TrustStoreItem { + private static final String FINGERPRINT_FIELD = "fingerprint"; + private static final String EXPIRY_FIELD = "expiry"; + + private final String fingerprint; + private final Instant expiry; + + public TrustStoreItem(String fingerprint, Instant expiry) { + this.fingerprint = fingerprint; + this.expiry = expiry; + } + + public String fingerprint() { + return fingerprint; + } + + public Instant expiry() { + return expiry; + } + + public void toSlime(Cursor trustedCertificatesRoot) { + Cursor object = trustedCertificatesRoot.addObject(); + object.setString(FINGERPRINT_FIELD, fingerprint); + object.setLong(EXPIRY_FIELD, expiry.toEpochMilli()); + } + + public static TrustStoreItem fromSlime(Inspector inspector) { + String fingerprint = inspector.field(FINGERPRINT_FIELD).asString(); + Instant expiry = Instant.ofEpochMilli(inspector.field(EXPIRY_FIELD).asLong()); + return new TrustStoreItem(fingerprint, expiry); + } + + @Override + public String toString() { + return "TrustedCertificate{" + + "fingerprint='" + fingerprint + '\'' + + ", expiry=" + expiry + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TrustStoreItem that = (TrustStoreItem) o; + return Objects.equals(fingerprint, that.fingerprint) && Objects.equals(expiry, that.expiry); + } + + @Override + public int hashCode() { + return Objects.hash(fingerprint, expiry); + } +}
\ No newline at end of file diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index d5a4c459ef3..7e252391cb3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -210,7 +210,7 @@ public class CuratorDatabaseClient { toState.isAllocated() ? node.allocation() : Optional.empty(), node.history().recordStateTransition(node.state(), toState, agent, clock.instant()), node.type(), node.reports(), node.modelName(), node.reservedTo(), - node.exclusiveToApplicationId(), node.exclusiveToClusterType(), node.switchHostname()); + node.exclusiveToApplicationId(), node.exclusiveToClusterType(), node.switchHostname(), node.trustedCertificates()); writeNode(toState, curatorTransaction, node, newNode); writtenNodes.add(newNode); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 19bbe92eff6..868837daeeb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -36,6 +36,7 @@ import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.OsVersion; import com.yahoo.vespa.hosted.provision.node.Reports; import com.yahoo.vespa.hosted.provision.node.Status; +import com.yahoo.vespa.hosted.provision.node.TrustStoreItem; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -95,6 +96,7 @@ public class NodeSerializer { private static final String exclusiveToApplicationIdKey = "exclusiveTo"; private static final String exclusiveToClusterTypeKey = "exclusiveToClusterType"; private static final String switchHostnameKey = "switchHostname"; + private static final String trustedCertificatesKey = "trustedCertificates"; // Node resource fields private static final String flavorKey = "flavor"; @@ -122,6 +124,10 @@ public class NodeSerializer { // Network port fields private static final String networkPortsKey = "networkPorts"; + // Trusted certificates fields + private static final String fingerprintKey = "fingerprint"; + private static final String expiresKey = "expires"; + // A cache of deserialized Node objects. The cache is keyed on the hash of serialized node data. // // Deserializing a Node from slime is expensive, and happens frequently. Node instances that have already been @@ -182,6 +188,7 @@ public class NodeSerializer { node.reservedTo().ifPresent(tenant -> object.setString(reservedToKey, tenant.value())); node.exclusiveToApplicationId().ifPresent(applicationId -> object.setString(exclusiveToApplicationIdKey, applicationId.serializedForm())); node.exclusiveToClusterType().ifPresent(clusterType -> object.setString(exclusiveToClusterTypeKey, clusterType.name())); + trustedCertificatesToSlime(node.trustedCertificates(), object.setArray(trustedCertificatesKey)); } private void toSlime(Flavor flavor, Cursor object) { @@ -236,6 +243,14 @@ public class NodeSerializer { }); } + private void trustedCertificatesToSlime(List<TrustStoreItem> trustStoreItems, Cursor array) { + trustStoreItems.forEach(cert -> { + Cursor object = array.addObject(); + object.setString(fingerprintKey, cert.fingerprint()); + object.setLong(expiresKey, cert.expiry().toEpochMilli()); + }); + } + // ---------------- Deserialization -------------------------------------------------- public Node fromJson(Node.State state, byte[] data) { @@ -269,7 +284,8 @@ public class NodeSerializer { reservedToFromSlime(object.field(reservedToKey)), exclusiveToApplicationIdFromSlime(object.field(exclusiveToApplicationIdKey)), exclusiveToClusterTypeFromSlime(object.field(exclusiveToClusterTypeKey)), - switchHostnameFromSlime(object.field(switchHostnameKey))); + switchHostnameFromSlime(object.field(switchHostnameKey)), + trustedCertificatesFromSlime(object)); } private Status statusFromSlime(Inspector object) { @@ -419,6 +435,13 @@ public class NodeSerializer { return Optional.of(ClusterSpec.Type.from(object.asString())); } + private List<TrustStoreItem> trustedCertificatesFromSlime(Inspector object) { + return SlimeUtils.entriesStream(object.field(trustedCertificatesKey)) + .map(elem -> new TrustStoreItem(elem.field(fingerprintKey).asString(), + Instant.ofEpochMilli(elem.field(expiresKey).asLong()))) + .collect(Collectors.toList()); + } + // ----------------- Enum <-> string mappings ---------------------------------------- /** Returns the event type, or null if this event type should be ignored */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index 8d37c13d2bc..5dfacc2c3d2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -25,6 +25,7 @@ import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.Report; import com.yahoo.vespa.hosted.provision.node.Reports; +import com.yahoo.vespa.hosted.provision.node.TrustStoreItem; import java.io.IOException; import java.io.InputStream; @@ -186,6 +187,8 @@ public class NodePatcher implements AutoCloseable { return node.withExclusiveToClusterType(SlimeUtils.optionalString(value).map(ClusterSpec.Type::valueOf).orElse(null)); case "switchHostname": return value.type() == Type.NIX ? node.withoutSwitchHostname() : node.withSwitchHostname(value.asString()); + case "trustStore": + return nodeWithTrustStore(node, value); default : throw new IllegalArgumentException("Could not apply field '" + name + "' on a node: No such modifiable field"); } @@ -230,6 +233,14 @@ public class NodePatcher implements AutoCloseable { return patchedNode; } + private Node nodeWithTrustStore(Node node, Inspector inspector) { + List<TrustStoreItem> trustStoreItems = + SlimeUtils.entriesStream(inspector) + .map(TrustStoreItem::fromSlime) + .collect(Collectors.toList()); + return node.with(trustStoreItems); + } + private Set<String> asStringSet(Inspector field) { if ( ! field.type().equals(Type.ARRAY)) throw new IllegalArgumentException("Expected an ARRAY value, got a " + field.type()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index 2cf671514c4..80100128379 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.History; +import com.yahoo.vespa.hosted.provision.node.TrustStoreItem; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.orchestrator.status.HostStatus; @@ -182,6 +183,7 @@ class NodesResponse extends SlimeJsonResponse { node.modelName().ifPresent(modelName -> object.setString("modelName", modelName)); node.switchHostname().ifPresent(switchHostname -> object.setString("switchHostname", switchHostname)); nodeRepository.archiveUris().archiveUriFor(node).ifPresent(uri -> object.setString("archiveUri", uri)); + trustedCertsToSlime(node.trustedCertificates(), object); } private void toSlime(ApplicationId id, Cursor object) { @@ -228,6 +230,12 @@ class NodesResponse extends SlimeJsonResponse { addresses.forEach(address -> addressesArray.addString(address.hostname())); } + private void trustedCertsToSlime(List<TrustStoreItem> trustStoreItems, Cursor object) { + if (trustStoreItems.isEmpty()) return; + Cursor array = object.setArray("trustStore"); + trustStoreItems.forEach(cert -> cert.toSlime(array)); + } + private String lastElement(String path) { if (path.endsWith("/")) path = path.substring(0, path.length()-1); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeListMicroBenchmarkTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeListMicroBenchmarkTest.java new file mode 100644 index 00000000000..a1edbec5769 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeListMicroBenchmarkTest.java @@ -0,0 +1,100 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision; + +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provisioning.FlavorsConfig; +import com.yahoo.vespa.hosted.provision.node.IP; +import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; +import org.junit.Test; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.Random; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * @author hmusum + */ +public class NodeListMicroBenchmarkTest { + + private static final NodeFlavors nodeFlavors = createNodeFlavors(); + private final NodeResources resources0 = new NodeResources(1, 30, 20, 1.5); + private int nodeCounter = 0; + private static final int hostCount = 1000; + + @Test + public void testChildrenOf() { + List<Node> nodes = createHosts(); + + List<Node> childNodes = nodes.stream().map(host -> createNodes(host.hostname())).flatMap(Collection::stream).collect(Collectors.toList()); + nodes.addAll(childNodes); + NodeList nodeList = new NodeList(nodes, false); + + int iterations = 100000; + Random random = new Random(0); + ArrayList<Integer> indexes = new ArrayList<>(); + for (int i = 0; i < iterations; i++) { + indexes.add(random.nextInt(hostCount)); + } + + Instant start = Instant.now(); + for (int i = 0; i < iterations; i++) { + nodeList.childrenOf(nodes.get(indexes.get(i))); + } + Duration duration = Duration.between(start, Instant.now()); + System.out.println("Calling NodeList.childrenOf took " + duration + " (" + duration.toNanos() / iterations / 1000 + " microseconds per invocation)"); + } + + private List<Node> createHosts() { + List<Node> hosts = new ArrayList<>(); + for (int i = 0; i < hostCount; i++) { + hosts.add(Node.create("host" + i, IP.Config.of(Set.of("::1"), createIps(), List.of()), + "host" + i, getFlavor("host"), NodeType.host).build()); + } + return hosts; + } + + private List<Node> createNodes(String parentHostname) { + List<Node> nodes = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + nodeCounter++; + Node node = Node.reserve(Set.of("::2"), "node" + nodeCounter, parentHostname, resources0, NodeType.tenant).build(); + nodes.add(node); + } + return nodes; + } + + private static NodeFlavors createNodeFlavors() { + FlavorConfigBuilder builder = new FlavorConfigBuilder(); + builder.addFlavor("host", 30, 30, 40, 3, Flavor.Type.BARE_METAL); + builder.addFlavor("node", 3, 3, 4, 3, Flavor.Type.DOCKER_CONTAINER); + FlavorsConfig flavorsConfig = builder.build(); + return new NodeFlavors(Optional.ofNullable(flavorsConfig).orElseGet(ProvisioningTester::createConfig)); + } + + private Flavor getFlavor(String name) { + return nodeFlavors.getFlavor(name).orElseThrow(() -> new RuntimeException("Unknown flavor")); + } + + private Set<String> createIps() { + // Allow 4 containers + int start = 2; + int count = 4; + Set<String> ipAddressPool = new LinkedHashSet<>(); + for (int i = start; i < (start + count); i++) { + ipAddressPool.add("::" + i); + } + return ipAddressPool; + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index 158a1d6e5ac..921b78252d3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -29,6 +29,7 @@ import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.Report; import com.yahoo.vespa.hosted.provision.node.Reports; +import com.yahoo.vespa.hosted.provision.node.TrustStoreItem; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import org.junit.Test; @@ -462,6 +463,16 @@ public class NodeSerializerTest { assertEquals(exclusiveToCluster, node.exclusiveToClusterType().get()); } + @Test + public void truststore_serialization() { + Node node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(createNode())); + assertEquals(List.of(), node.trustedCertificates()); + List<TrustStoreItem> trustStoreItems = List.of(new TrustStoreItem("foo", Instant.parse("2023-09-01T23:59:59Z")), new TrustStoreItem("bar", Instant.parse("2025-05-20T23:59:59Z"))); + node = node.with(trustStoreItems); + node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(node)); + assertEquals(trustStoreItems, node.trustedCertificates()); + } + private byte[] createNodeJson(String hostname, String... ipAddress) { String ipAddressJsonPart = ""; if (ipAddress.length > 0) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index fa68533bb06..4c9597d00bd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -43,7 +43,6 @@ import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import com.yahoo.vespa.hosted.provision.testutils.MockProvisionServiceProvider; -import com.yahoo.vespa.orchestrator.OrchestrationException; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.service.duper.ConfigServerApplication; @@ -687,11 +686,7 @@ public class ProvisioningTester { Orchestrator orchestrator = Optional.ofNullable(this.orchestrator) .orElseGet(() -> { Orchestrator orch = mock(Orchestrator.class); - try { - doThrow(new RuntimeException()).when(orch).acquirePermissionToRemove(any()); - } catch (OrchestrationException e) { - throw new RuntimeException(e); - } + doThrow(new RuntimeException()).when(orch).acquirePermissionToRemove(any()); return orch; }); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 6c052a6c364..88a8a22913e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -1023,6 +1023,26 @@ public class NodesV2ApiTest { tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "archiveUri", false); } + @Test + public void trusted_certificates_patch() throws IOException { + String url = "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"; + tester.assertPartialResponse(new Request(url), "\"trustStore\":[]", false); // initially empty list + + String trustStore = "\"trustStore\":[" + + "{" + + "\"fingerprint\":\"foo\"," + + "\"expiry\":1632302251000" + + "}," + + "{" + + "\"fingerprint\":\"bar\"," + + "\"expiry\":1758532706000" + + "}" + + "]"; + assertResponse(new Request(url, Utf8.toBytes("{"+trustStore+"}"), Request.Method.PATCH), + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + tester.assertPartialResponse(new Request(url), trustStore, true); + } + private static String asDockerNodeJson(String hostname, String parentHostname, String... ipAddress) { return asDockerNodeJson(hostname, NodeType.tenant, parentHostname, ipAddress); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestrationException.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestrationException.java index 95d862d9e72..f2eae278150 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestrationException.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestrationException.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.orchestrator; import java.util.Arrays; -public class OrchestrationException extends Exception { +public class OrchestrationException extends RuntimeException { public OrchestrationException(Throwable cause) { super(cause); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java index fd115702588..47ba6dedd84 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java @@ -17,13 +17,20 @@ public interface ClusterApi { String clusterInfo(); ClusterId clusterId(); ServiceType serviceType(); + + /** Some human-readable string naming the service(s) to a human reader. */ + String serviceDescription(boolean plural); + boolean isStorageCluster(); - /** Returns the reasons no services are up in the implied group, or empty if some services are up. */ - Optional<SuspensionReasons> reasonsForNoServicesInGroupIsUp(); + boolean isConfigServerLike(); + + /** Returns the non-empty reasons for why all services are down, or otherwise empty. */ + Optional<SuspensionReasons> allServicesDown(); + boolean noServicesOutsideGroupIsDown() throws HostStateChangeDeniedException; - int percentageOfServicesDown(); + int percentageOfServicesDownOutsideGroup(); int percentageOfServicesDownIfGroupIsAllowedToBeDown(); Optional<StorageNode> storageNodeInGroup(); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java index 00cb65a09b0..6880700e4a9 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java @@ -19,7 +19,6 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Collections; -import java.util.Comparator; import java.util.HashSet; import java.util.Map; import java.util.Optional; @@ -41,9 +40,10 @@ class ClusterApiImpl implements ClusterApi { private final ClusterControllerClientFactory clusterControllerClientFactory; private final Clock clock; private final Set<ServiceInstance> servicesInGroup; - private final Set<ServiceInstance> servicesDownInGroup; private final Set<ServiceInstance> servicesNotInGroup; - private final Set<ServiceInstance> servicesDownAndNotInGroup; + + /** Lazily initialized in servicesDownAndNotInGroup(), do not access directly. */ + private Set<ServiceInstance> servicesDownAndNotInGroup = null; /* * There are two sources for the number of config servers in a cluster. The config server config and the node @@ -81,9 +81,6 @@ class ClusterApiImpl implements ClusterApi { servicesInGroup = serviceInstancesByLocality.getOrDefault(true, Collections.emptySet()); servicesNotInGroup = serviceInstancesByLocality.getOrDefault(false, Collections.emptySet()); - servicesDownInGroup = servicesInGroup.stream().filter(this::serviceEffectivelyDown).collect(Collectors.toSet()); - servicesDownAndNotInGroup = servicesNotInGroup.stream().filter(this::serviceEffectivelyDown).collect(Collectors.toSet()); - int serviceInstances = serviceCluster.serviceInstances().size(); if (clusterParams.size().isPresent() && serviceInstances < clusterParams.size().getAsInt()) { missingServices = clusterParams.size().getAsInt() - serviceInstances; @@ -110,6 +107,11 @@ class ClusterApiImpl implements ClusterApi { } @Override + public String serviceDescription(boolean plural) { + return serviceCluster.serviceDescription(plural); + } + + @Override public boolean isStorageCluster() { return VespaModelUtil.isStorage(serviceCluster); } @@ -120,7 +122,12 @@ class ClusterApiImpl implements ClusterApi { } @Override - public Optional<SuspensionReasons> reasonsForNoServicesInGroupIsUp() { + public boolean isConfigServerLike() { + return serviceCluster.isConfigServerLike(); + } + + @Override + public Optional<SuspensionReasons> allServicesDown() { SuspensionReasons reasons = new SuspensionReasons(); for (ServiceInstance service : servicesInGroup) { @@ -156,29 +163,18 @@ class ClusterApiImpl implements ClusterApi { @Override public boolean noServicesOutsideGroupIsDown() throws HostStateChangeDeniedException { - Optional<ServiceInstance> serviceWithUnknownStatus = servicesNotInGroup - .stream() - .filter(serviceInstance -> serviceInstance.serviceStatus() == ServiceStatus.UNKNOWN) - .min(Comparator.comparing(ServiceInstance::descriptiveName)); - if (serviceWithUnknownStatus.isPresent()) { - throw new HostStateChangeDeniedException( - nodeGroup, - HostedVespaPolicy.UNKNOWN_SERVICE_STATUS, - "Service status of " + serviceWithUnknownStatus.get().descriptiveName() + " is not yet known"); - } - - return servicesDownAndNotInGroup.size() + missingServices == 0; + return servicesDownAndNotInGroup().size() + missingServices == 0; } @Override - public int percentageOfServicesDown() { - int numberOfServicesDown = servicesDownAndNotInGroup.size() + missingServices + servicesDownInGroup.size(); + public int percentageOfServicesDownOutsideGroup() { + int numberOfServicesDown = servicesDownAndNotInGroup().size() + missingServices; return numberOfServicesDown * 100 / (serviceCluster.serviceInstances().size() + missingServices); } @Override public int percentageOfServicesDownIfGroupIsAllowedToBeDown() { - int numberOfServicesDown = servicesDownAndNotInGroup.size() + missingServices + servicesInGroup.size(); + int numberOfServicesDown = servicesDownAndNotInGroup().size() + missingServices + servicesInGroup.size(); return numberOfServicesDown * 100 / (serviceCluster.serviceInstances().size() + missingServices); } @@ -200,15 +196,14 @@ class ClusterApiImpl implements ClusterApi { description.append(" "); final int nodeLimit = 3; - description.append("Suspended hosts: "); description.append(suspended.stream().sorted().distinct().limit(nodeLimit).collect(Collectors.toList()).toString()); if (suspended.size() > nodeLimit) { - description.append(", and " + (suspended.size() - nodeLimit) + " more"); + description.append(" and " + (suspended.size() - nodeLimit) + " others"); } - description.append("."); + description.append(" are suspended."); } - Set<ServiceInstance> downElsewhere = servicesDownAndNotInGroup.stream() + Set<ServiceInstance> downElsewhere = servicesDownAndNotInGroup().stream() .filter(serviceInstance -> !suspended.contains(serviceInstance.hostName())) .collect(Collectors.toSet()); @@ -217,7 +212,6 @@ class ClusterApiImpl implements ClusterApi { description.append(" "); final int serviceLimit = 2; // services info is verbose - description.append("Services down on resumed hosts: "); description.append(Stream.concat( downElsewhere.stream().map(ServiceInstance::toString).sorted(), missingServices > 0 ? Stream.of(descriptionOfMissingServices) : Stream.of()) @@ -226,9 +220,9 @@ class ClusterApiImpl implements ClusterApi { .toString()); if (downElsewhereTotal > serviceLimit) { - description.append(", and " + (downElsewhereTotal - serviceLimit) + " more"); + description.append(" and " + (downElsewhereTotal - serviceLimit) + " others"); } - description.append("."); + description.append(" are down."); } return description.toString(); @@ -288,19 +282,31 @@ class ClusterApiImpl implements ClusterApi { return "{ clusterId=" + clusterId() + ", serviceType=" + serviceType() + " }"; } + private Set<ServiceInstance> servicesDownAndNotInGroup() { + if (servicesDownAndNotInGroup == null) { + servicesDownAndNotInGroup = servicesNotInGroup.stream().filter(this::serviceEffectivelyDown).collect(Collectors.toSet()); + } + return servicesDownAndNotInGroup; + } + private HostStatus hostStatus(HostName hostName) { return hostInfos.getOrNoRemarks(hostName).status(); } - private boolean serviceEffectivelyDown(ServiceInstance service) { + private boolean serviceEffectivelyDown(ServiceInstance service) throws HostStateChangeDeniedException { if (hostStatus(service.hostName()).isSuspended()) { return true; } - if (service.serviceStatus() == ServiceStatus.DOWN) { - return true; + switch (service.serviceStatus()) { + case DOWN: return true; + case UNKNOWN: + throw new HostStateChangeDeniedException( + nodeGroup, + HostedVespaPolicy.UNKNOWN_SERVICE_STATUS, + "Service status of " + service.descriptiveName() + " is not yet known"); + default: + return false; } - - return false; } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java index 208e12690ff..fe06ef7d75b 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java @@ -23,6 +23,16 @@ public class HostedVespaClusterPolicy implements ClusterPolicy { @Override public SuspensionReasons verifyGroupGoingDownIsFine(ClusterApi clusterApi) throws HostStateChangeDeniedException { + return verifyGroupGoingDownIsFine(clusterApi, false); + } + + @Override + public void verifyGroupGoingDownPermanentlyIsFine(ClusterApi clusterApi) throws HostStateChangeDeniedException { + verifyGroupGoingDownIsFine(clusterApi, true); + } + + private SuspensionReasons verifyGroupGoingDownIsFine(ClusterApi clusterApi, boolean permanent) + throws HostStateChangeDeniedException { if (clusterApi.noServicesOutsideGroupIsDown()) { return SuspensionReasons.nothingNoteworthy(); } @@ -32,47 +42,28 @@ public class HostedVespaClusterPolicy implements ClusterPolicy { return SuspensionReasons.nothingNoteworthy(); } - Optional<SuspensionReasons> suspensionReasons = clusterApi.reasonsForNoServicesInGroupIsUp(); - if (suspensionReasons.isPresent()) { - return suspensionReasons.get(); + // Be a bit more cautious when removing nodes permanently + if (!permanent) { + // Disallow suspending a 2nd and downed config server to avoid losing ZK quorum. + if (!clusterApi.isConfigServerLike()) { + Optional<SuspensionReasons> suspensionReasons = clusterApi.allServicesDown(); + if (suspensionReasons.isPresent()) { + return suspensionReasons.get(); + } + } } String message = percentageOfServicesAllowedToBeDown <= 0 - ? "Suspension of service with type '" + clusterApi.serviceType() + "' not allowed: " - + clusterApi.percentageOfServicesDown() + "% are suspended already." + clusterApi.downDescription() - : "Suspension of service with type '" + clusterApi.serviceType() - + "' would increase from " + clusterApi.percentageOfServicesDown() - + "% to " + clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() - + "%, over the limit of " + percentageOfServicesAllowedToBeDown + "%." - + clusterApi.downDescription(); + ? clusterApi.percentageOfServicesDownOutsideGroup() + "% of the " + clusterApi.serviceDescription(true) + + " are down or suspended already:" + clusterApi.downDescription() + : "The percentage of downed or suspended " + clusterApi.serviceDescription(true) + + " would increase from " + clusterApi.percentageOfServicesDownOutsideGroup() + "% to " + + clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() + "% (limit is " + + percentageOfServicesAllowedToBeDown + "%):" + clusterApi.downDescription(); throw new HostStateChangeDeniedException(clusterApi.getNodeGroup(), ENOUGH_SERVICES_UP_CONSTRAINT, message); } - @Override - public void verifyGroupGoingDownPermanentlyIsFine(ClusterApi clusterApi) throws HostStateChangeDeniedException { - // This policy is similar to verifyGroupGoingDownIsFine, except that having no services up in the group will - // not allow the suspension: We are a bit more cautious when removing nodes. - - if (clusterApi.noServicesOutsideGroupIsDown()) { - return; - } - - int percentageOfServicesAllowedToBeDown = getConcurrentSuspensionLimit(clusterApi) - .asPercentage(); - if (clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() <= percentageOfServicesAllowedToBeDown) { - return; - } - - throw new HostStateChangeDeniedException( - clusterApi.getNodeGroup(), - ENOUGH_SERVICES_UP_CONSTRAINT, - "Down percentage for service type " + clusterApi.serviceType() - + " would increase to " + clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() - + "%, over the limit of " + percentageOfServicesAllowedToBeDown + "%." - + clusterApi.downDescription()); - } - // Non-private for testing purposes ConcurrentSuspensionLimitForCluster getConcurrentSuspensionLimit(ClusterApi clusterApi) { // Possible service clusters on a node as of 2021-01-22: diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java index 83c902ed981..f885d5301de 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java @@ -102,7 +102,7 @@ public class OrchestratorTest { fail(); } catch (HostStateChangeDeniedException e) { assertThat(e.getMessage(), containsString("Changing the state of cfg2 would violate enough-services-up")); - assertThat(e.getMessage(), containsString("Suspended hosts: [cfg1]")); + assertThat(e.getMessage(), containsString("[cfg1] are suspended.")); } // cfg1 is removed from the application @@ -114,7 +114,7 @@ public class OrchestratorTest { fail(); } catch (HostStateChangeDeniedException e) { assertThat(e.getMessage(), containsString("Changing the state of cfg2 would violate enough-services-up")); - assertThat(e.getMessage(), containsString("Services down on resumed hosts: [1 missing config server]")); + assertThat(e.getMessage(), containsString("[1 missing config server] are down.")); } // cfg1 is reprovisioned, added to the node repo, and activated @@ -129,7 +129,7 @@ public class OrchestratorTest { fail(); } catch (HostStateChangeDeniedException e) { assertThat(e.getMessage(), containsString("Changing the state of cfg1 would violate enough-services-up")); - assertThat(e.getMessage(), containsString("Suspended hosts: [cfg2]")); + assertThat(e.getMessage(), containsString("[cfg2] are suspended")); } // etc (should be the same as for cfg1) diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java index da8591c6631..3ea739b385e 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java @@ -95,11 +95,11 @@ public class ClusterApiImplTest { assertEquals("{ clusterId=cluster, serviceType=service-type }", clusterApi.clusterInfo()); assertFalse(clusterApi.isStorageCluster()); - assertEquals(" Suspended hosts: [host3, host4]. Services down on resumed hosts: [" + - "ServiceInstance{configId=service-2, hostName=host2, serviceStatus=" + - "ServiceStatusInfo{status=DOWN, since=Optional.empty, lastChecked=Optional.empty}}].", + assertEquals(" [host3, host4] are suspended. [ServiceInstance{configId=service-2, hostName=host2, " + + "serviceStatus=ServiceStatusInfo{status=DOWN, since=Optional.empty, lastChecked=Optional.empty}}] " + + "are down.", clusterApi.downDescription()); - assertEquals(60, clusterApi.percentageOfServicesDown()); + assertEquals(60, clusterApi.percentageOfServicesDownOutsideGroup()); assertEquals(80, clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()); } @@ -181,8 +181,8 @@ public class ClusterApiImplTest { fail(); } catch (HostStateChangeDeniedException e) { assertThat(e.getMessage(), - containsString("Suspension of service with type 'configserver' not allowed: 33% are suspended already. " + - "Services down on resumed hosts: [1 missing config server].")); + containsString("Changing the state of cfg1 would violate enough-services-up: 33% of the config " + + "servers are down or suspended already: [1 missing config server] are down.")); } } @@ -201,18 +201,25 @@ public class ClusterApiImplTest { fail(); } catch (HostStateChangeDeniedException e) { assertThat(e.getMessage(), - containsString("Suspension of service with type 'hostadmin' not allowed: 33% are suspended already. " + - "Services down on resumed hosts: [1 missing config server host].")); + containsString("Changing the state of cfg1 would violate enough-services-up: 33% of the config " + + "server hosts are down or suspended already: [1 missing config server host] are down.")); } } @Test - public void testCfg1SuspendsIfDownWithMissingCfg3() throws HostStateChangeDeniedException { + public void testCfg1DoesNotSuspendIfDownWithMissingCfg3() throws HostStateChangeDeniedException { ClusterApiImpl clusterApi = makeCfg1ClusterApi(ServiceStatus.DOWN, ServiceStatus.UP); HostedVespaClusterPolicy policy = new HostedVespaClusterPolicy(flagSource, zone); - policy.verifyGroupGoingDownIsFine(clusterApi); + try { + policy.verifyGroupGoingDownIsFine(clusterApi); + fail(); + } catch (HostStateChangeDeniedException e) { + assertThat(e.getMessage(), + containsString("Changing the state of cfg1 would violate enough-services-up: 33% of the config " + + "servers are down or suspended already: [1 missing config server] are down.")); + } } @Test @@ -320,7 +327,7 @@ public class ClusterApiImplTest { clock); assertEquals(expectedNoServicesInGroupIsUp.map(SuspensionReasons::getMessagesInOrder), - clusterApi.reasonsForNoServicesInGroupIsUp().map(SuspensionReasons::getMessagesInOrder)); + clusterApi.allServicesDown().map(SuspensionReasons::getMessagesInOrder)); assertEquals(expectedNoServicesOutsideGroupIsDown, clusterApi.noServicesOutsideGroupIsDown()); } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java index 303dabebba8..8f47051621f 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java @@ -131,13 +131,14 @@ public class HostedVespaClusterPolicyTest { int percentageOfServicesDownIfGroupIsAllowedToBeDown, boolean expectSuccess) throws HostStateChangeDeniedException { when(clusterApi.noServicesOutsideGroupIsDown()).thenReturn(noServicesOutsideGroupIsDown); - when(clusterApi.reasonsForNoServicesInGroupIsUp()).thenReturn(noServicesInGroupIsUp); + when(clusterApi.allServicesDown()).thenReturn(noServicesInGroupIsUp); when(clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()).thenReturn(20); doReturn(ConcurrentSuspensionLimitForCluster.TEN_PERCENT).when(policy).getConcurrentSuspensionLimit(clusterApi); when(applicationApi.applicationId()).thenReturn(ApplicationId.fromSerializedForm("a:b:c")); when(clusterApi.serviceType()).thenReturn(new ServiceType("service-type")); - when(clusterApi.percentageOfServicesDown()).thenReturn(5); + when(clusterApi.serviceDescription(true)).thenReturn("services of {service-type,cluster-id}"); + when(clusterApi.percentageOfServicesDownOutsideGroup()).thenReturn(5); when(clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()).thenReturn(percentageOfServicesDownIfGroupIsAllowedToBeDown); when(clusterApi.downDescription()).thenReturn(" Down description"); @@ -152,9 +153,10 @@ public class HostedVespaClusterPolicyTest { } } catch (HostStateChangeDeniedException e) { if (!expectSuccess) { - assertEquals("Changing the state of node-group would violate enough-services-up: " + - "Suspension of service with type 'service-type' would increase from 5% to 13%, " + - "over the limit of 10%. Down description", e.getMessage()); + assertEquals("Changing the state of node-group would violate enough-services-up: The percentage of downed " + + "or suspended services of {service-type,cluster-id} would increase from 5% to 13% (limit is 10%): " + + "Down description", + e.getMessage()); assertEquals("enough-services-up", e.getConstraintName()); } } diff --git a/parent/pom.xml b/parent/pom.xml index dd13971968d..23ba49ccb6e 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -567,18 +567,6 @@ <version>${prometheus.client.version}</version> </dependency> <dependency> - <!-- TODO: Try to remove, as this overlaps with javax.activation. --> - <groupId>jakarta.activation</groupId> - <artifactId>jakarta.activation-api</artifactId> - <version>1.2.1</version> - </dependency> - <dependency> - <!-- TODO: Try to remove, as this conflicts with javax.xml.bind:jaxb-api --> - <groupId>jakarta.xml.bind</groupId> - <artifactId>jakarta.xml.bind-api</artifactId> - <version>2.3.2</version> - </dependency> - <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>4.12</version> @@ -94,6 +94,7 @@ <module>jdisc_jetty</module> <module>jrt</module> <module>linguistics</module> + <module>linguistics-components</module> <module>logd</module> <module>logserver</module> <module>messagebus</module> diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index c0353747e80..36ef347b7ac 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -54,6 +54,7 @@ vespa_define_module( src/apps/vespa-transactionlog-inspect TESTS + src/tests/bmcluster/estimate_moved_docs_ratio src/tests/grouping src/tests/proton/attribute src/tests/proton/attribute/attribute_aspect_delayer diff --git a/searchcore/src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp b/searchcore/src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp index e5c3959d2d4..0227d9539d2 100644 --- a/searchcore/src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp +++ b/searchcore/src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp @@ -17,6 +17,8 @@ #include <vespa/searchcore/bmcluster/bm_node_stats_reporter.h> #include <vespa/searchcore/bmcluster/bm_range.h> #include <vespa/searchcore/bmcluster/bucket_selector.h> +#include <vespa/searchcore/bmcluster/calculate_moved_docs_ratio.h> +#include <vespa/searchcore/bmcluster/estimate_moved_docs_ratio.h> #include <vespa/searchcore/bmcluster/spi_bm_feed_handler.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/vespalib/io/fileutil.h> @@ -51,6 +53,8 @@ using search::bmcluster::BmNode; using search::bmcluster::BmNodeStatsReporter; using search::bmcluster::BmRange; using search::bmcluster::BucketSelector; +using search::bmcluster::CalculateMovedDocsRatio; +using search::bmcluster::EstimateMovedDocsRatio; using search::index::DummyFileHeaderContext; using storage::lib::State; @@ -100,76 +104,6 @@ vespalib::string& get_mode_name(Mode mode) { return (i < mode_names.size()) ? mode_names[i] : bad_mode_name; } -double -estimate_lost_docs_base_ratio(uint32_t redundancy, uint32_t lost_nodes, uint32_t num_nodes) -{ - if (redundancy > lost_nodes) { - return 0.0; - } - double loss_ratio = 1.0; - for (uint32_t i = 0; i < redundancy; ++i) { - loss_ratio *= ((double) (lost_nodes - i)) / (num_nodes - i); - } - LOG(info, "estimated lost docs base ratio: %4.2f", loss_ratio); - return loss_ratio; -} - -double -estimate_moved_docs_ratio_grow(uint32_t redundancy, uint32_t added_nodes, uint32_t num_nodes) -{ - double new_redundancy = redundancy; - double new_per_node_doc_ratio = new_redundancy / num_nodes; - double moved_ratio = new_per_node_doc_ratio * added_nodes; - LOG(info, "estimated_moved_docs_ratio_grow(%u,%u,%u)=%4.2f", redundancy, added_nodes, num_nodes, moved_ratio); - return moved_ratio; -} - -double -estimate_moved_docs_ratio_shrink(uint32_t redundancy, uint32_t retired_nodes, uint32_t num_nodes) -{ - double old_redundancy = redundancy; - double old_per_node_doc_ratio = old_redundancy / num_nodes; - uint32_t new_nodes = num_nodes - retired_nodes; - double new_redundancy = std::min(redundancy, new_nodes); - double new_per_node_doc_ratio = new_redundancy / new_nodes; - double moved_ratio = (new_per_node_doc_ratio - old_per_node_doc_ratio) * new_nodes; - LOG(info, "estimated_moved_docs_ratio_shrink(%u,%u,%u)=%4.2f", redundancy, retired_nodes, num_nodes, moved_ratio); - return moved_ratio; -} - -double -estimate_moved_docs_ratio_crash(uint32_t redundancy, uint32_t crashed_nodes, uint32_t num_nodes) -{ - double old_redundancy = redundancy; - double old_per_node_doc_ratio = old_redundancy / num_nodes; - uint32_t new_nodes = num_nodes - crashed_nodes; - double new_redundancy = std::min(redundancy, new_nodes); - double new_per_node_doc_ratio = new_redundancy / new_nodes; - double lost_docs_ratio = estimate_lost_docs_base_ratio(redundancy, crashed_nodes, num_nodes) * new_redundancy; - double moved_ratio = (new_per_node_doc_ratio - old_per_node_doc_ratio) * new_nodes - lost_docs_ratio; - LOG(info, "estimated_moved_docs_ratio_crash(%u,%u,%u)=%4.2f", redundancy, crashed_nodes, num_nodes, moved_ratio); - return moved_ratio; -} - -double -estimate_moved_docs_ratio_replace(uint32_t redundancy, uint32_t added_nodes, uint32_t retired_nodes, uint32_t num_nodes) -{ - uint32_t old_nodes = num_nodes - added_nodes; - double old_redundancy = std::min(redundancy, old_nodes); - double old_per_node_doc_ratio = old_redundancy / old_nodes; - uint32_t new_nodes = num_nodes - retired_nodes; - double new_redundancy = std::min(redundancy, new_nodes); - double new_per_node_doc_ratio = new_redundancy / new_nodes; - double moved_ratio = new_per_node_doc_ratio * added_nodes; - uint32_t stable_nodes = num_nodes - added_nodes - retired_nodes; - // Account for extra documents moved from retired nodes to stable nodes - double extra_per_stable_node_doc_ratio = new_per_node_doc_ratio * added_nodes / old_nodes; - double extra_moved_ratio = (std::min(1.0, new_per_node_doc_ratio + extra_per_stable_node_doc_ratio) - old_per_node_doc_ratio) * stable_nodes; - moved_ratio += extra_moved_ratio; - LOG(info, "estimated_moved_docs_ratio_replace(%u,%u,%u,%u)=%4.2f, (of which %4.2f extra)", redundancy, added_nodes, retired_nodes, num_nodes, moved_ratio, extra_moved_ratio); - return moved_ratio; -} - class BMParams : public BmClusterParams, public BmFeedParams { @@ -391,7 +325,7 @@ Benchmark::estimate_lost_docs() case Mode::TEMP_CRASH: { double new_redundancy = std::min(_params.get_redundancy(), _params.get_num_nodes() - _params.get_flip_nodes()); - auto lost_docs_ratio = estimate_lost_docs_base_ratio(_params.get_redundancy(), _params.get_flip_nodes(), _params.get_num_nodes()) * new_redundancy; + auto lost_docs_ratio = EstimateMovedDocsRatio().estimate_lost_docs_base_ratio(_params.get_redundancy(), _params.get_flip_nodes(), _params.get_num_nodes()) * new_redundancy; return _params.get_documents() * lost_docs_ratio; } default: @@ -404,14 +338,21 @@ Benchmark::estimate_moved_docs() { switch(_params.get_mode()) { case Mode::GROW: - return _params.get_documents() * estimate_moved_docs_ratio_grow(_params.get_redundancy(), _params.get_flip_nodes(), _params.get_num_nodes()); + return _params.get_documents() * EstimateMovedDocsRatio().estimate_moved_docs_ratio_grow(_params.get_redundancy(), _params.get_flip_nodes(), _params.get_num_nodes()); case Mode::SHRINK: - return _params.get_documents() * estimate_moved_docs_ratio_shrink(_params.get_redundancy(), _params.get_flip_nodes(), _params.get_num_nodes()); + return _params.get_documents() * EstimateMovedDocsRatio().estimate_moved_docs_ratio_shrink(_params.get_redundancy(), _params.get_flip_nodes(), _params.get_num_nodes()); case Mode::PERM_CRASH: case Mode::TEMP_CRASH: - return _params.get_documents() * estimate_moved_docs_ratio_crash(_params.get_redundancy(), _params.get_flip_nodes(), _params.get_num_nodes()); + return _params.get_documents() * EstimateMovedDocsRatio().estimate_moved_docs_ratio_crash(_params.get_redundancy(), _params.get_flip_nodes(), _params.get_num_nodes()); case Mode::REPLACE: - return _params.get_documents() * estimate_moved_docs_ratio_replace(_params.get_redundancy(), _params.get_flip_nodes(), _params.get_flip_nodes(), _params.get_num_nodes()); + if (_params.get_num_nodes() < 10) { + // Calculate better estimate for moved docs ratio with brute force + auto scanner = CalculateMovedDocsRatio::make_replace_calculator(_params.get_redundancy(), _params.get_flip_nodes(), _params.get_flip_nodes(), _params.get_num_nodes()); + scanner.scan(); + return _params.get_documents() * scanner.get_moved_docs_ratio(); + } else { + return _params.get_documents() * EstimateMovedDocsRatio().estimate_moved_docs_ratio_replace(_params.get_redundancy(), _params.get_flip_nodes(), _params.get_flip_nodes(), _params.get_num_nodes()); + } default: return 0.0; } diff --git a/searchcore/src/tests/bmcluster/estimate_moved_docs_ratio/CMakeLists.txt b/searchcore/src/tests/bmcluster/estimate_moved_docs_ratio/CMakeLists.txt new file mode 100644 index 00000000000..23f72e4f1fa --- /dev/null +++ b/searchcore/src/tests/bmcluster/estimate_moved_docs_ratio/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_bmcluster_estimate_moved_docs_ratio_test_app TEST + SOURCES + estimate_moved_docs_ratio_test.cpp + DEPENDS + searchcore_bmcluster + GTest::GTest +) + +vespa_add_test(NAME searchcore_bmcluster_estimate_moved_docs_ratio_test_app + COMMAND searchcore_bmcluster_estimate_moved_docs_ratio_test_app) diff --git a/searchcore/src/tests/bmcluster/estimate_moved_docs_ratio/estimate_moved_docs_ratio_test.cpp b/searchcore/src/tests/bmcluster/estimate_moved_docs_ratio/estimate_moved_docs_ratio_test.cpp new file mode 100644 index 00000000000..79af31e3247 --- /dev/null +++ b/searchcore/src/tests/bmcluster/estimate_moved_docs_ratio/estimate_moved_docs_ratio_test.cpp @@ -0,0 +1,134 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/searchcore/bmcluster/calculate_moved_docs_ratio.h> +#include <vespa/searchcore/bmcluster/estimate_moved_docs_ratio.h> +#include <iostream> + +#include <vespa/log/log.h> +LOG_SETUP("estimate_moved_docs_ratio_test"); + +using search::bmcluster::CalculateMovedDocsRatio; +using search::bmcluster::EstimateMovedDocsRatio; + +namespace { + +bool verbose; + +TEST(EstimateMovedDocsRatioTest, estimate_lost_docs_ratio) +{ + for (uint32_t nodes = 1; nodes < 2; ++nodes) { + for (uint32_t redundancy = 1; redundancy <= nodes; ++redundancy) { + for (uint32_t lost_nodes = 0; lost_nodes <= nodes; ++lost_nodes) { + auto scanner = CalculateMovedDocsRatio::make_crash_calculator(redundancy, lost_nodes, nodes); + scanner.scan(); + double lost_docs_base_ratio = scanner.get_lost_docs_base_ratio(); + double estimated_lost_docs_base_ratio = EstimateMovedDocsRatio().estimate_lost_docs_base_ratio(redundancy, lost_nodes, nodes); + EXPECT_DOUBLE_EQ(lost_docs_base_ratio, estimated_lost_docs_base_ratio); + } + } + } +} + +TEST(EstimateMovedDocsRatioTest, estimate_moved_docs_ratio_grow) +{ + for (uint32_t nodes = 1; nodes < 10; ++nodes) { + for (uint32_t redundancy = 1; redundancy <= nodes; ++redundancy) { + for (uint32_t added_nodes = 0; added_nodes <= nodes; ++added_nodes) { + auto scanner = CalculateMovedDocsRatio::make_grow_calculator(redundancy, added_nodes, nodes); + scanner.scan(); + double moved_docs_ratio = scanner.get_moved_docs_ratio(); + double estimated_moved_docs_ratio = EstimateMovedDocsRatio().estimate_moved_docs_ratio_grow(redundancy, added_nodes, nodes); + EXPECT_DOUBLE_EQ(moved_docs_ratio, estimated_moved_docs_ratio); + } + } + } +} + +TEST(EstimateMovedDocsRatioTest, estimate_moved_docs_ratio_shrink) +{ + for (uint32_t nodes = 1; nodes < 10; ++nodes) { + for (uint32_t redundancy = 1; redundancy <= nodes; ++redundancy) { + for (uint32_t retired_nodes = 0; retired_nodes <= nodes; ++retired_nodes) { + auto scanner = CalculateMovedDocsRatio::make_shrink_calculator(redundancy, retired_nodes, nodes); + scanner.scan(); + double moved_docs_ratio = scanner.get_moved_docs_ratio(); + double estimated_moved_docs_ratio = EstimateMovedDocsRatio().estimate_moved_docs_ratio_shrink(redundancy, retired_nodes, nodes); + EXPECT_DOUBLE_EQ(moved_docs_ratio, estimated_moved_docs_ratio); + } + } + } +} + +TEST(EstimateMovedDocsRatioTest, estimate_moved_docs_ratio_crash) +{ + double epsilon = 1e-15; + for (uint32_t nodes = 1; nodes < 10; ++nodes) { + for (uint32_t redundancy = 1; redundancy <= nodes; ++redundancy) { + for (uint32_t crashed_nodes = 0; crashed_nodes <= nodes; ++crashed_nodes) { + auto scanner = CalculateMovedDocsRatio::make_crash_calculator(redundancy, crashed_nodes, nodes); + scanner.scan(); + double moved_docs_ratio = scanner.get_moved_docs_ratio(); + double estimated_moved_docs_ratio = EstimateMovedDocsRatio().estimate_moved_docs_ratio_crash(redundancy, crashed_nodes, nodes); + EXPECT_NEAR(moved_docs_ratio, estimated_moved_docs_ratio, epsilon); + } + } + } +} + +TEST(EstimateMovedDocsRatioTest, estimate_moved_docs_ratio_replace) +{ + uint32_t bad_cases = 0; + uint32_t really_bad_cases = 0; + if (verbose) { + std::cout << "Summary: HDR Red A Ret N Act Est ScaleMv ScaleEs States" << std::endl; + } + for (uint32_t nodes = 1; nodes < 6; ++nodes) { + for (uint32_t redundancy = 1; redundancy <= nodes; ++redundancy) { + for (uint32_t retired_nodes = 0; retired_nodes <= nodes; ++retired_nodes) { + for (uint32_t added_nodes = 0; added_nodes <= nodes - retired_nodes; ++added_nodes) { + // std::cout << "Estimate moved docs ratio replace " << retired_nodes << " of " << nodes << " retired, added " << added_nodes << " nodes ,redundancy " << redundancy << std::endl; + auto scanner = CalculateMovedDocsRatio::make_replace_calculator(redundancy, added_nodes, retired_nodes, nodes); + scanner.scan(); + double moved_docs_ratio = scanner.get_moved_docs_ratio(); + double estimated_moved_docs_ratio = EstimateMovedDocsRatio(verbose).estimate_moved_docs_ratio_replace(redundancy, added_nodes, retired_nodes, nodes); + double error_ratio = abs(moved_docs_ratio - estimated_moved_docs_ratio); + bool bad = error_ratio > 1e-8; + bool really_bad = error_ratio > 0.2 * estimated_moved_docs_ratio + 1e-8; + if (bad) { + ++bad_cases; + } + if (really_bad) { + ++really_bad_cases; + } + if (verbose) { + double scaled_moved = moved_docs_ratio * scanner.get_checked_states(); + double scaled_estimated_moved = estimated_moved_docs_ratio * scanner.get_checked_states(); + std::cout << "Summary: " << (bad ? "BAD" : "OK ") << std::setw(4) << redundancy << std::setw(4) << added_nodes << std::setw(4) << retired_nodes << std::setw(4) << nodes << " " << std::setw(12) << std::setprecision(5) << std::fixed << moved_docs_ratio << " " << std::setw(12) << std::setprecision(5) << std::fixed << estimated_moved_docs_ratio << " " << std::setw(12) << std::setprecision(5) << std::fixed << scaled_moved << " " << std::setw(12) << std::setprecision(5) << std::fixed << scaled_estimated_moved << std::setw(8) << scanner.get_checked_states(); + std::cout << " ["; + for (uint32_t node_idx = 0; node_idx < nodes; ++node_idx) { + std::cout << std::setw(8) << scanner.get_moved_docs_per_node()[node_idx]; + } + std::cout << " ]" << std::endl; + } + // TODO: Fix calculation so we get zero bad cases. + // EXPECT_DOUBLE_EQ(moved_docs_ratio, estimated_moved_docs_ratio); + } + } + } + } + EXPECT_LE(6, bad_cases); + EXPECT_LE(1, really_bad_cases); +} + +} // namespace + +int +main(int argc, char* argv[]) +{ + ::testing::InitGoogleTest(&argc, argv); + if (argc > 1 && std::string("--verbose") == argv[1]) { + verbose = true; + } + return RUN_ALL_TESTS(); +} diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp index b7f217290d6..1475ac7f9d7 100644 --- a/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp +++ b/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp @@ -62,7 +62,13 @@ TEST_F(DiskMemUsageSamplerTest, resource_usage_is_sampled) std::this_thread::sleep_for(50ms); } LOG(info, "Polled %zu times (%zu ms) to get a sample", i, i * 50); +#ifdef __linux__ + // Anonymous resident memory used by current process is sampled. EXPECT_GT(filter().getMemoryStats().getAnonymousRss(), 0); +#else + // Anonymous resident memory used by current process is not sampled. + EXPECT_EQ(filter().getMemoryStats().getAnonymousRss(), 0); +#endif EXPECT_GT(filter().getDiskUsedSize(), 0); EXPECT_EQ(100, filter().get_transient_memory_usage()); EXPECT_EQ(100.0 / memory_size_bytes, filter().get_relative_transient_memory_usage()); diff --git a/searchcore/src/vespa/searchcore/bmcluster/CMakeLists.txt b/searchcore/src/vespa/searchcore/bmcluster/CMakeLists.txt index d2ab8b0c2a9..0de021a9514 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/bmcluster/CMakeLists.txt @@ -19,7 +19,9 @@ vespa_add_library(searchcore_bmcluster STATIC bm_storage_link.cpp bm_storage_message_addresses.cpp bucket_info_queue.cpp + calculate_moved_docs_ratio.cpp document_api_message_bus_bm_feed_handler.cpp + estimate_moved_docs_ratio.cpp pending_tracker.cpp pending_tracker_hash.cpp spi_bm_feed_handler.cpp diff --git a/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.cpp b/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.cpp index 2a214130392..f31f5fb6b88 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.cpp +++ b/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.cpp @@ -6,7 +6,7 @@ namespace search::bmcluster { BmClusterParams::BmClusterParams() - : _bucket_db_stripe_bits(0), + : _bucket_db_stripe_bits(4), _distributor_stripes(0), _enable_distributor(false), _enable_service_layer(false), diff --git a/searchcore/src/vespa/searchcore/bmcluster/bm_node_stats_reporter.cpp b/searchcore/src/vespa/searchcore/bmcluster/bm_node_stats_reporter.cpp index 83956dfc274..ecd0593031e 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bm_node_stats_reporter.cpp +++ b/searchcore/src/vespa/searchcore/bmcluster/bm_node_stats_reporter.cpp @@ -95,16 +95,16 @@ BmNodeStatsReporter::report() for (auto &node : node_stats) { auto& document_db = node.get_document_db_stats(); if (document_db.has_value()) { - s << Width(8) << document_db.value().get_total_docs(); + s << Width(10) << document_db.value().get_total_docs(); } else { - s << Width(8) << "-"; + s << Width(10) << "-"; } totals += node; } if (totals.get_document_db_stats().has_value()) { - s << Width(8) << totals.get_document_db_stats().value().get_total_docs(); + s << Width(10) << totals.get_document_db_stats().value().get_total_docs(); } else { - s << Width(8) << "-"; + s << Width(10) << "-"; } auto& total_buckets = totals.get_buckets_stats(); if (total_buckets.has_value()) { diff --git a/searchcore/src/vespa/searchcore/bmcluster/calculate_moved_docs_ratio.cpp b/searchcore/src/vespa/searchcore/bmcluster/calculate_moved_docs_ratio.cpp new file mode 100644 index 00000000000..63f23476a95 --- /dev/null +++ b/searchcore/src/vespa/searchcore/bmcluster/calculate_moved_docs_ratio.cpp @@ -0,0 +1,142 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "calculate_moved_docs_ratio.h" +#include <cassert> + +namespace search::bmcluster { + +namespace { + +uint32_t make_bit_range(uint32_t low, uint32_t high) +{ + return (1u << high) - (1u << low); +} + +} + +struct CalculateMovedDocsRatio::Placements +{ + uint32_t _mask; + uint32_t _count; + + Placements() + : _mask(0u), + _count(0u) + { + } + + Placements(uint32_t mask, uint32_t count) noexcept + : _mask(mask), + _count(count) + { + } + + Placements add(uint32_t idx) const noexcept { + return Placements(_mask | (1u << idx), _count + 1); + } + + Placements add(uint32_t idx, uint32_t mask, uint32_t redundancy) const noexcept { + return (((_count < redundancy) && ((1u << idx) & mask) != 0)) ? Placements(_mask | (1u << idx), _count + 1) : *this; + } +}; + + +CalculateMovedDocsRatio::CalculateMovedDocsRatio(uint32_t nodes, uint32_t redundancy, uint32_t old_placement_mask, uint32_t new_placement_mask, uint32_t new_up_mask) + : _num_states(nodes + 1), + _nodes(nodes), + _old_placement_mask(old_placement_mask), + _new_placement_mask(new_placement_mask), + _new_up_mask(new_up_mask), + _moved_docs(0u), + _moved_docs_per_node(nodes), + _checked_states(0u), + _lost_docs_base(0u), + _redundancy(redundancy), + _old_redundancy(std::min(redundancy, (uint32_t)__builtin_popcount(old_placement_mask))), + _new_redundancy(std::min(redundancy, (uint32_t)__builtin_popcount(new_placement_mask))) +{ + assert((new_placement_mask & ~new_up_mask) == 0u); + uint32_t states = 1; + for (uint32_t level = nodes; level > 0; --level) { + states *= std::max(1u, nodes - level); + _num_states[level] = states; + } + _num_states[0] = states * nodes; +} + +CalculateMovedDocsRatio::~CalculateMovedDocsRatio() = default; + +CalculateMovedDocsRatio +CalculateMovedDocsRatio::make_grow_calculator(uint32_t redundancy, uint32_t added_nodes, uint32_t nodes) +{ + uint32_t old_placement_mask = make_bit_range(added_nodes, nodes); + uint32_t new_placement_mask = make_bit_range(0, nodes); + return CalculateMovedDocsRatio(nodes, redundancy, old_placement_mask, new_placement_mask, new_placement_mask); +} + +CalculateMovedDocsRatio +CalculateMovedDocsRatio::make_shrink_calculator(uint32_t redundancy, uint32_t retired_nodes, uint32_t nodes) +{ + uint32_t old_placement_mask = make_bit_range(0, nodes); + uint32_t new_placement_mask = make_bit_range(retired_nodes, nodes); + return CalculateMovedDocsRatio(nodes, redundancy, old_placement_mask, new_placement_mask, old_placement_mask); +} + +CalculateMovedDocsRatio +CalculateMovedDocsRatio::make_crash_calculator(uint32_t redundancy, uint32_t crashed_nodes, uint32_t nodes) +{ + uint32_t old_placement_mask = make_bit_range(0, nodes); + uint32_t new_placement_mask = make_bit_range(crashed_nodes, nodes); + return CalculateMovedDocsRatio(nodes, redundancy, old_placement_mask, new_placement_mask, new_placement_mask); + +} + +CalculateMovedDocsRatio +CalculateMovedDocsRatio::make_replace_calculator(uint32_t redundancy, uint32_t added_nodes, uint32_t retired_nodes, uint32_t nodes) +{ + uint32_t old_placement_mask = make_bit_range(added_nodes, nodes); + uint32_t new_placement_mask = make_bit_range(added_nodes + retired_nodes, nodes) | make_bit_range(0, added_nodes); + uint32_t new_up_mask = make_bit_range(0, nodes); + return CalculateMovedDocsRatio(nodes, redundancy, old_placement_mask, new_placement_mask, new_up_mask); +} + +void +CalculateMovedDocsRatio::scan(Placements selected, Placements old_placement, Placements new_placement) +{ + if (old_placement._count >= _old_redundancy) { + if ((old_placement._mask & _new_up_mask) == 0) { + _lost_docs_base += _num_states[selected._count]; + _checked_states += _num_states[selected._count]; + return; + } + if (new_placement._count >= _new_redundancy) { + _checked_states += _num_states[selected._count]; + uint32_t only_new_mask = new_placement._mask & ~old_placement._mask; + if (only_new_mask != 0) { + _moved_docs += _num_states[selected._count] * (uint32_t)__builtin_popcount(only_new_mask); + for (uint32_t node_idx = 0; node_idx < _nodes; ++node_idx) { + if ((only_new_mask & (1u << node_idx)) != 0) { + _moved_docs_per_node[node_idx] += _num_states[selected._count]; + } + } + } + return; + } + } + assert(selected._count < _nodes); + for (uint32_t node_idx = 0; node_idx < _nodes; ++node_idx) { + if ((selected._mask & (1u << node_idx)) != 0) { + continue; + } + scan(selected.add(node_idx), old_placement.add(node_idx, _old_placement_mask, _old_redundancy), new_placement.add(node_idx, _new_placement_mask, _new_redundancy)); + } +} + +void +CalculateMovedDocsRatio::scan() +{ + scan(Placements(), Placements(), Placements()); + assert(_checked_states == _num_states[0]); +} + +} diff --git a/searchcore/src/vespa/searchcore/bmcluster/calculate_moved_docs_ratio.h b/searchcore/src/vespa/searchcore/bmcluster/calculate_moved_docs_ratio.h new file mode 100644 index 00000000000..cd161177246 --- /dev/null +++ b/searchcore/src/vespa/searchcore/bmcluster/calculate_moved_docs_ratio.h @@ -0,0 +1,48 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstdint> +#include <vector> + +namespace search::bmcluster { + +/* + * Class for calculating moved docs ratio during + * document redistribution. + */ +class CalculateMovedDocsRatio +{ + class Placements; + std::vector<uint32_t> _num_states; + uint32_t _nodes; + uint32_t _old_placement_mask; + uint32_t _new_placement_mask; + uint32_t _new_up_mask; + uint32_t _moved_docs; + std::vector<uint32_t> _moved_docs_per_node; + uint32_t _checked_states; + uint32_t _lost_docs_base; + uint32_t _redundancy; + uint32_t _old_redundancy; + uint32_t _new_redundancy; + + void scan(Placements selected, Placements old_placement, Placements new_placement); +public: + CalculateMovedDocsRatio(uint32_t nodes, uint32_t redundancy, uint32_t old_placement_mask, uint32_t new_placement_mask, uint32_t new_up_mask); + ~CalculateMovedDocsRatio(); + static CalculateMovedDocsRatio make_grow_calculator(uint32_t redundancy, uint32_t added_nodes, uint32_t nodes); + static CalculateMovedDocsRatio make_shrink_calculator(uint32_t redundancy, uint32_t retired_nodes, uint32_t nodes); + static CalculateMovedDocsRatio make_crash_calculator(uint32_t redundancy, uint32_t crashed_nodes, uint32_t nodes); + static CalculateMovedDocsRatio make_replace_calculator(uint32_t redundancy, uint32_t added_nodes, uint32_t retired_nodes, uint32_t nodes); + void scan(); + uint32_t get_lost_docs_base() const noexcept { return _lost_docs_base; } + uint32_t get_checked_states() const noexcept { return _checked_states; } + uint32_t get_new_redundancy() const noexcept { return _new_redundancy; } + uint32_t get_moved_docs() const noexcept { return _moved_docs; } + const std::vector<uint32_t>& get_moved_docs_per_node() const noexcept { return _moved_docs_per_node; } + double get_lost_docs_base_ratio() const noexcept { return ((double) _lost_docs_base) / _checked_states; } + double get_moved_docs_ratio() const noexcept { return ((double) _moved_docs) / _checked_states; } +}; + +} diff --git a/searchcore/src/vespa/searchcore/bmcluster/estimate_moved_docs_ratio.cpp b/searchcore/src/vespa/searchcore/bmcluster/estimate_moved_docs_ratio.cpp new file mode 100644 index 00000000000..cffa6fc79e9 --- /dev/null +++ b/searchcore/src/vespa/searchcore/bmcluster/estimate_moved_docs_ratio.cpp @@ -0,0 +1,114 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "estimate_moved_docs_ratio.h" + +#include <vespa/log/log.h> +LOG_SETUP(".bmcluster.estimate_moved_docs_ratio"); + +namespace search::bmcluster { + +EstimateMovedDocsRatio::EstimateMovedDocsRatio() + : EstimateMovedDocsRatio(false) +{ +} + +EstimateMovedDocsRatio::EstimateMovedDocsRatio(bool verbose) + : _verbose(verbose) +{ +} + +double +EstimateMovedDocsRatio::estimate_lost_docs_base_ratio(uint32_t redundancy, uint32_t lost_nodes, uint32_t num_nodes) +{ + if (redundancy > lost_nodes) { + return 0.0; + } + double loss_ratio = 1.0; + for (uint32_t i = 0; i < redundancy; ++i) { + loss_ratio *= ((double) (lost_nodes - i)) / (num_nodes - i); + } + if (_verbose) { + LOG(info, "estimated lost docs base ratio: %4.2f", loss_ratio); + } + return loss_ratio; +} + +double +EstimateMovedDocsRatio::estimate_moved_docs_ratio_grow(uint32_t redundancy, uint32_t added_nodes, uint32_t num_nodes) +{ + if (added_nodes == num_nodes) { + return 0.0; + } + double new_redundancy = redundancy; + double new_per_node_doc_ratio = new_redundancy / num_nodes; + double moved_ratio = new_per_node_doc_ratio * added_nodes; + if (_verbose) { + LOG(info, "estimated_moved_docs_ratio_grow(%u,%u,%u)=%4.2f", redundancy, added_nodes, num_nodes, moved_ratio); + } + return moved_ratio; +} + +double +EstimateMovedDocsRatio::estimate_moved_docs_ratio_shrink(uint32_t redundancy, uint32_t retired_nodes, uint32_t num_nodes) +{ + if (retired_nodes == num_nodes) { + return 0.0; + } + double old_redundancy = redundancy; + double old_per_node_doc_ratio = old_redundancy / num_nodes; + uint32_t new_nodes = num_nodes - retired_nodes; + double new_redundancy = std::min(redundancy, new_nodes); + double new_per_node_doc_ratio = new_redundancy / new_nodes; + double moved_ratio = (new_per_node_doc_ratio - old_per_node_doc_ratio) * new_nodes; + if (_verbose) { + LOG(info, "estimated_moved_docs_ratio_shrink(%u,%u,%u)=%4.2f", redundancy, retired_nodes, num_nodes, moved_ratio); + } + return moved_ratio; +} + +double +EstimateMovedDocsRatio::estimate_moved_docs_ratio_crash(uint32_t redundancy, uint32_t crashed_nodes, uint32_t num_nodes) +{ + if (crashed_nodes == num_nodes) { + return 0.0; + } + double old_redundancy = redundancy; + double old_per_node_doc_ratio = old_redundancy / num_nodes; + uint32_t new_nodes = num_nodes - crashed_nodes; + double new_redundancy = std::min(redundancy, new_nodes); + double new_per_node_doc_ratio = new_redundancy / new_nodes; + double lost_docs_ratio = estimate_lost_docs_base_ratio(redundancy, crashed_nodes, num_nodes) * new_redundancy; + double moved_ratio = (new_per_node_doc_ratio - old_per_node_doc_ratio) * new_nodes - lost_docs_ratio; + if (_verbose) { + LOG(info, "estimated_moved_docs_ratio_crash(%u,%u,%u)=%4.2f", redundancy, crashed_nodes, num_nodes, moved_ratio); + } + return moved_ratio; +} + +double +EstimateMovedDocsRatio::estimate_moved_docs_ratio_replace(uint32_t redundancy, uint32_t added_nodes, uint32_t retired_nodes, uint32_t num_nodes) +{ + if (added_nodes == num_nodes || retired_nodes == num_nodes) { + return 0.0; + } + uint32_t old_nodes = num_nodes - added_nodes; + double old_redundancy = std::min(redundancy, old_nodes); + [[maybe_unused]] double old_per_node_doc_ratio = old_redundancy / old_nodes; + uint32_t new_nodes = num_nodes - retired_nodes; + double new_redundancy = std::min(redundancy, new_nodes); + double new_per_node_doc_ratio = new_redundancy / new_nodes; + double moved_ratio = new_per_node_doc_ratio * added_nodes; + uint32_t stable_nodes = num_nodes - added_nodes - retired_nodes; + // Account for extra documents moved from retired nodes to stable nodes + // TODO: Fix calculation + double baseline_per_node_doc_ratio = ((double) redundancy) / num_nodes; + double extra_per_stable_node_doc_ratio = std::min(baseline_per_node_doc_ratio * retired_nodes / new_nodes, 1.0 - old_per_node_doc_ratio); + double extra_moved_ratio = extra_per_stable_node_doc_ratio * stable_nodes; + moved_ratio += extra_moved_ratio; + if (_verbose) { + LOG(info, "estimated_moved_docs_ratio_replace(%u,%u,%u,%u)=%4.2f, (of which %4.2f extra)", redundancy, added_nodes, retired_nodes, num_nodes, moved_ratio, extra_moved_ratio); + } + return moved_ratio; +} + +} diff --git a/searchcore/src/vespa/searchcore/bmcluster/estimate_moved_docs_ratio.h b/searchcore/src/vespa/searchcore/bmcluster/estimate_moved_docs_ratio.h new file mode 100644 index 00000000000..fb14e80098d --- /dev/null +++ b/searchcore/src/vespa/searchcore/bmcluster/estimate_moved_docs_ratio.h @@ -0,0 +1,25 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstdint> + +namespace search::bmcluster { + +/* + * Class for estimating moved docs ratio during + * document redistribution. + */ +class EstimateMovedDocsRatio { + bool _verbose; +public: + EstimateMovedDocsRatio(); + explicit EstimateMovedDocsRatio(bool verbose); + double estimate_lost_docs_base_ratio(uint32_t redundancy, uint32_t lost_nodes, uint32_t num_nodes); + double estimate_moved_docs_ratio_grow(uint32_t redundancy, uint32_t added_nodes, uint32_t num_nodes); + double estimate_moved_docs_ratio_shrink(uint32_t redundancy, uint32_t retired_nodes, uint32_t num_nodes); + double estimate_moved_docs_ratio_crash(uint32_t redundancy, uint32_t crashed_nodes, uint32_t num_nodes); + double estimate_moved_docs_ratio_replace(uint32_t redundancy, uint32_t added_nodes, uint32_t retired_nodes, uint32_t num_nodes); +}; + +} diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index e8e3a3cb133..2468fd0c5c7 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -1034,6 +1034,7 @@ "public static final int LDEXP", "public static final int POW", "public static final int BIT", + "public static final int HAMMING", "public static final int MAP", "public static final int REDUCE", "public static final int JOIN", @@ -1389,7 +1390,8 @@ "public static final enum com.yahoo.searchlib.rankingexpression.rule.Function max", "public static final enum com.yahoo.searchlib.rankingexpression.rule.Function min", "public static final enum com.yahoo.searchlib.rankingexpression.rule.Function pow", - "public static final enum com.yahoo.searchlib.rankingexpression.rule.Function bit" + "public static final enum com.yahoo.searchlib.rankingexpression.rule.Function bit", + "public static final enum com.yahoo.searchlib.rankingexpression.rule.Function hamming" ] }, "com.yahoo.searchlib.rankingexpression.rule.FunctionNode": { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java index e41732f9d16..33ba3c6ef4b 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java @@ -157,6 +157,7 @@ public class TensorValue extends Value { case fmod: return value.fmod(argument); case ldexp: return value.ldexp(argument); case bit: return value.bit(argument); + case hamming: return value.hamming(argument); default: throw new UnsupportedOperationException("Cannot combine two tensors using " + function); } } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/Function.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/Function.java index 16aa947986d..3958711f74b 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/Function.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/Function.java @@ -46,7 +46,8 @@ public enum Function implements Serializable { max(2) { public double evaluate(double x, double y) { return max(x,y); } }, min(2) { public double evaluate(double x, double y) { return min(x,y); } }, pow(2) { public double evaluate(double x, double y) { return pow(x,y); } }, - bit(2) { public double evaluate(double x, double y) { return ((int)y < 8 && (int)y >= 0 && ((int)x & (1 << (int)y)) != 0) ? 1.0 : 0.0; } }; + bit(2) { public double evaluate(double x, double y) { return ((int)y < 8 && (int)y >= 0 && ((int)x & (1 << (int)y)) != 0) ? 1.0 : 0.0; } }, + hamming(2) { public double evaluate(double x, double y) { return ScalarFunctions.Hamming.hamming(x, y); } }; private final int arity; diff --git a/searchlib/src/main/javacc/RankingExpressionParser.jj b/searchlib/src/main/javacc/RankingExpressionParser.jj index 99eff010628..7bfbfd6c005 100755 --- a/searchlib/src/main/javacc/RankingExpressionParser.jj +++ b/searchlib/src/main/javacc/RankingExpressionParser.jj @@ -124,6 +124,7 @@ TOKEN : // MIN <POW: "pow"> | <BIT: "bit"> | + <HAMMING: "hamming"> | <MAP: "map"> | <REDUCE: "reduce"> | @@ -735,7 +736,8 @@ Function binaryFunctionName() : { } <MAX> { return Function.max; } | <MIN> { return Function.min; } | <POW> { return Function.pow; } | - <BIT> { return Function.bit; } + <BIT> { return Function.bit; } | + <HAMMING> { return Function.hamming; } } List<ExpressionNode> expressionList() : diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java index 4a3c4b248be..246dbcb2b1e 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java @@ -288,6 +288,8 @@ public class EvaluationTestCase { tester.assertEvaluates("{ {h:0}:1.5, {h:1}:1.5 }", "0.5 + tensor0", "{ {h:0}:1.0,{h:1}:1.0 }"); tester.assertEvaluates("{ {x:0,y:0}:0, {x:1,y:0}:0 }", "atan2(tensor0, tensor1)", "{ {x:0}:0, {x:1}:0 }", "{ {y:0}:1 }"); + tester.assertEvaluates("{ {x:0,y:0}:2, {x:1,y:0}:7 }", + "hamming(tensor0, tensor1)", "{ {x:0}:97, {x:1}:-1 }", "{ {y:0}:1 }"); tester.assertEvaluates("{ {x:0,y:0}:0, {x:1,y:0}:1 }", "tensor0 > tensor1", "{ {x:0}:3, {x:1}:7 }", "{ {y:0}:5 }"); tester.assertEvaluates("{ {x:0,y:0}:1, {x:1,y:0}:0 }", diff --git a/searchlib/src/tests/rankingexpression/rankingexpressionlist b/searchlib/src/tests/rankingexpression/rankingexpressionlist index 77b2294c668..d41570732d9 100644 --- a/searchlib/src/tests/rankingexpression/rankingexpressionlist +++ b/searchlib/src/tests/rankingexpression/rankingexpressionlist @@ -87,6 +87,7 @@ floor(10) relu(10) sigmoid(10) atan2(10, 20); atan2(10,20) +hamming(42, -16); hamming(42,-16) ldexp(10, 20); ldexp(10,20) pow(10, 20); pow(10,20) fmod(10, 20); fmod(10,20) diff --git a/searchlib/src/vespa/searchlib/engine/search_protocol_proto.h b/searchlib/src/vespa/searchlib/engine/search_protocol_proto.h index e2eeca74e44..bb9887f6e74 100644 --- a/searchlib/src/vespa/searchlib/engine/search_protocol_proto.h +++ b/searchlib/src/vespa/searchlib/engine/search_protocol_proto.h @@ -7,6 +7,6 @@ #pragma GCC diagnostic ignored "-Wsuggest-override" #endif -#include "search_protocol.pb.h" +#include <vespa/searchlib/engine/search_protocol.pb.h> #pragma GCC diagnostic pop diff --git a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp index 0a13b196c43..90369009f0e 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp +++ b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp @@ -3,8 +3,10 @@ #include "blueprintresolver.h" #include "blueprintfactory.h" #include "featurenameparser.h" +#include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/util/threadstackexecutor.h> #include <stack> #include <cassert> #include <set> @@ -14,6 +16,8 @@ LOG_SETUP(".fef.blueprintresolver"); using vespalib::make_string_short::fmt; +using vespalib::ThreadStackExecutor; +using vespalib::makeLambdaTask; namespace search::fef { @@ -271,8 +275,7 @@ BlueprintResolver::compile() { assert(_executorSpecs.empty()); // only one compilation allowed Compiler compiler(_factory, _indexEnv, _executorSpecs, _featureMap); - std::thread compile_thread([&]() - { + auto compile_task = makeLambdaTask([&]() { compiler.probe_stack(); for (const auto &seed: _seeds) { auto ref = compiler.resolve_feature(seed, Blueprint::AcceptInput::ANY); @@ -282,7 +285,10 @@ BlueprintResolver::compile() _seedMap.emplace(FeatureNameParser(seed).featureName(), ref); } }); - compile_thread.join(); + ThreadStackExecutor executor(1, 8_Mi); + executor.execute(std::move(compile_task)); + executor.sync(); + executor.shutdown(); size_t stack_usage = compiler.stack_usage(); if (stack_usage > (128_Ki)) { LOG(warning, "high stack usage: %zu bytes", stack_usage); diff --git a/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java b/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java index 215dc311af3..dbabf1274af 100644 --- a/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java +++ b/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java @@ -21,10 +21,13 @@ import java.io.UncheckedIOException; import java.math.BigInteger; import java.security.GeneralSecurityException; import java.security.KeyPair; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.security.PrivateKey; import java.security.PublicKey; import java.security.Signature; import java.security.SignatureException; +import java.security.cert.CertificateEncodingException; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.time.Duration; @@ -178,4 +181,16 @@ public class X509CertificateUtils { .build(); return new X509CertificateWithKey(cert, keyPair.getPrivate()); } + + /** + * @return certificate SHA-1 fingerprint + */ + public static byte[] getX509CertificateFingerPrint(X509Certificate certificate) { + try { + MessageDigest sha1 = MessageDigest.getInstance("SHA-1"); + return sha1.digest(certificate.getEncoded()); + } catch (CertificateEncodingException | NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } } diff --git a/slobrok/src/apps/slobrok/slobrok.cpp b/slobrok/src/apps/slobrok/slobrok.cpp index b2748762a12..18a401e1db6 100644 --- a/slobrok/src/apps/slobrok/slobrok.cpp +++ b/slobrok/src/apps/slobrok/slobrok.cpp @@ -50,7 +50,6 @@ App::Main() { uint32_t portnum = 2773; vespalib::string cfgId; - bool useNewLogic = false; int argi = 1; const char* optArg; @@ -64,7 +63,7 @@ App::Main() portnum = atoi(optArg); break; case 'N': - useNewLogic = true; + // ignored break; default: LOG(error, "unknown option letter '%c'", c); @@ -76,11 +75,11 @@ App::Main() if (cfgId.empty()) { LOG(debug, "no config id specified"); ConfigShim shim(portnum); - mainobj = std::make_unique<SBEnv>(shim, useNewLogic); + mainobj = std::make_unique<SBEnv>(shim); } else { ConfigShim shim(portnum, cfgId); shim.enableStateServer(true); - mainobj = std::make_unique<SBEnv>(shim, useNewLogic); + mainobj = std::make_unique<SBEnv>(shim); } hook_sigterm(); res = mainobj->MainLoop(); diff --git a/slobrok/src/tests/rpc_mapping_monitor/rpc_mapping_monitor_test.cpp b/slobrok/src/tests/rpc_mapping_monitor/rpc_mapping_monitor_test.cpp index f503453f934..0edcde172aa 100644 --- a/slobrok/src/tests/rpc_mapping_monitor/rpc_mapping_monitor_test.cpp +++ b/slobrok/src/tests/rpc_mapping_monitor/rpc_mapping_monitor_test.cpp @@ -148,7 +148,7 @@ TEST_F(RpcMappingMonitorTest, hurry_means_faster) { EXPECT_TRUE(debugger.step_until([&]() { return ((hist.map[foo_a].samples() > 0)); })); auto t2 = debugger.time(); - fprintf(stderr, "hurry: ~%zu ms, normal: ~%zu ms\n", count_ms(t1-t0), count_ms(t2-t0)); + fprintf(stderr, "hurry: ~%" PRIu64 " ms, normal: ~%" PRIu64 " ms\n", count_ms(t1-t0), count_ms(t2-t0)); EXPECT_GT((t2 - t0), 10 * (t1 - t0)); EXPECT_EQ(hist.map[foo_a].state(), State::UP); EXPECT_EQ(hist.map[baz_b].state(), State::UP); @@ -218,7 +218,7 @@ TEST_F(RpcMappingMonitorTest, detect_ping_interval) { a.set_last_conn(nullptr); EXPECT_TRUE(debugger.step_until([&]() { return (a.last_conn); })); auto t2 = debugger.time(); - fprintf(stderr, "ping interval: ~%zu ms\n", count_ms(t2-t1)); + fprintf(stderr, "ping interval: ~%" PRIu64 " ms\n", count_ms(t2-t1)); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/slobrok/src/vespa/slobrok/server/CMakeLists.txt b/slobrok/src/vespa/slobrok/server/CMakeLists.txt index ae1a05c5181..5168758e46d 100644 --- a/slobrok/src/vespa/slobrok/server/CMakeLists.txt +++ b/slobrok/src/vespa/slobrok/server/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(slobrok_slobrokserver SOURCES - cmd.cpp configshim.cpp exchange_manager.cpp i_monitored_server.cpp @@ -23,8 +22,6 @@ vespa_add_library(slobrok_slobrokserver request_completion_handler.cpp reserved_name.cpp rpc_mapping_monitor.cpp - rpc_server_manager.cpp - rpc_server_map.cpp rpchooks.cpp rpcmirror.cpp sbenv.cpp diff --git a/slobrok/src/vespa/slobrok/server/cmd.cpp b/slobrok/src/vespa/slobrok/server/cmd.cpp deleted file mode 100644 index df856189d89..00000000000 --- a/slobrok/src/vespa/slobrok/server/cmd.cpp +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - - -#include "cmd.h" -#include "rpc_server_map.h" -#include "reserved_name.h" -#include "remote_slobrok.h" -#include "sbenv.h" - -#include <vespa/log/log.h> -LOG_SETUP(".slobrok.server.cmd"); - -namespace slobrok { - -//----------------------------------------------------------------------------- - -struct ScriptData { - SBEnv &env; - const std::string name; - const std::string spec; - FRT_RPCRequest * const registerRequest; - - enum { - RDC_INIT, XCH_WANTADD, CHK_RPCSRV, XCH_DOADD, XCH_IGNORE, RDC_INVAL - } _state; - - ScriptData(SBEnv &e, const std::string &n, const std::string &s, FRT_RPCRequest *r) - : env(e), name(n), spec(s), registerRequest(r), _state(RDC_INIT) - {} -}; - -//----------------------------------------------------------------------------- - -const std::string & -ScriptCommand::name() { return _data->name; } - -const std::string & -ScriptCommand::spec() { return _data->spec; } - -ScriptCommand::ScriptCommand(std::unique_ptr<ScriptData> data) - : _data(std::move(data)) -{} - -ScriptCommand::ScriptCommand(ScriptCommand &&) = default; -ScriptCommand& -ScriptCommand::operator= (ScriptCommand &&) = default; -ScriptCommand::~ScriptCommand() = default; - -ScriptCommand -ScriptCommand::makeRegRpcSrvCmd(SBEnv &env, - const std::string &name, const std::string &spec, - FRT_RPCRequest *req) -{ - return ScriptCommand(std::make_unique<ScriptData>(env, name, spec, req)); -} - -ScriptCommand -ScriptCommand::makeIgnoreCmd(SBEnv &env, const std::string & name, const std::string &spec) -{ - auto data = std::make_unique<ScriptData>(env, name, spec, nullptr); - data->_state = ScriptData::XCH_IGNORE; - return ScriptCommand(std::move(data)); -} - -ScriptCommand -ScriptCommand::makeRegCompleter(SBEnv &env, - const std::string &name, const std::string &spec, - FRT_RPCRequest *req) -{ - auto data = std::make_unique<ScriptData>(env, name, spec, req); - data->_state = ScriptData::XCH_DOADD; - return ScriptCommand(std::move(data)); -} - -void -ScriptCommand::doRequest() -{ - LOG_ASSERT(_data->_state == ScriptData::RDC_INIT); - doneHandler(OkState()); -} - -void cleanupReservation(ScriptData & data) -{ - RpcServerMap &map = data.env.rpcServerMap(); - const ReservedName *rsvp = map.getReservation(data.name.c_str()); - if (rsvp != nullptr && rsvp->isLocal) { - map.removeReservation(data.name.c_str()); - } -} - -void -ScriptCommand::doneHandler(OkState result) -{ - LOG_ASSERT(_data); - std::unique_ptr<ScriptData> dataUP = std::move(_data); - LOG_ASSERT(! _data); - ScriptData & data = *dataUP; - const char *name_p = data.name.c_str(); - const char *spec_p = data.spec.c_str(); - ExchangeManager &xch = data.env.exchangeManager(); - RpcServerManager &rsm = data.env.rpcServerManager(); - - if (result.failed()) { - LOG(warning, "failed [%s->%s] in state %d: %s", name_p, spec_p, data._state, result.errorMsg.c_str()); - if (data._state != ScriptData::XCH_IGNORE) { - cleanupReservation(data); - } - // XXX should handle different state errors differently? - if (data.registerRequest != nullptr) { - data.registerRequest->SetError(FRTE_RPC_METHOD_FAILED, result.errorMsg.c_str()); - data.registerRequest->Return(); - } else { - LOG(warning, "ignored: %s", result.errorMsg.c_str()); - } - return; - } - if (data._state == ScriptData::RDC_INIT) { - LOG(spam, "phase wantAdd(%s,%s)", name_p, spec_p); - data._state = ScriptData::XCH_WANTADD; - xch.wantAdd(std::move(dataUP)); - return; - } else if (data._state == ScriptData::XCH_WANTADD) { - LOG(spam, "phase addManaged(%s,%s)", name_p, spec_p); - data._state = ScriptData::CHK_RPCSRV; - rsm.addManaged(std::move(dataUP)); - return; - } else if (data._state == ScriptData::CHK_RPCSRV) { - LOG(spam, "phase doAdd(%s,%s)", name_p, spec_p); - data._state = ScriptData::XCH_DOADD; - xch.doAdd(std::move(dataUP)); - return; - } else if (data._state == ScriptData::XCH_DOADD) { - LOG(debug, "done doAdd(%s,%s)", name_p, spec_p); - data._state = ScriptData::RDC_INVAL; - // all OK - if (data.registerRequest != nullptr) { - data.registerRequest->Return(); - } - cleanupReservation(data); - return; - } else if (data._state == ScriptData::XCH_IGNORE) { - return; - } - // no other state should be possible - LOG_ABORT("should not be reached"); -} - -//----------------------------------------------------------------------------- - -} // namespace slobrok diff --git a/slobrok/src/vespa/slobrok/server/cmd.h b/slobrok/src/vespa/slobrok/server/cmd.h deleted file mode 100644 index e7f42f75e42..00000000000 --- a/slobrok/src/vespa/slobrok/server/cmd.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "ok_state.h" -#include <memory> - -class FRT_RPCRequest; - -namespace slobrok { - -class SBEnv; -struct ScriptData; - -class ScriptCommand -{ -private: - std::unique_ptr<ScriptData> _data; - ScriptCommand(std::unique_ptr<ScriptData> data); -public: - const std::string &name(); - const std::string &spec(); - - ScriptCommand(ScriptCommand &&); - ScriptCommand& operator= (ScriptCommand &&); - ~ScriptCommand(); - - static ScriptCommand makeRegRpcSrvCmd(SBEnv &env, const std::string &name, const std::string &spec, FRT_RPCRequest *req); - static ScriptCommand makeIgnoreCmd(SBEnv &env, const std::string &name, const std::string &spec); - static ScriptCommand makeRegCompleter(SBEnv &env, const std::string &name, const std::string &spec, FRT_RPCRequest *req); - - void doneHandler(OkState result); - void doRequest(); -}; - -} // namespace slobrok diff --git a/slobrok/src/vespa/slobrok/server/exchange_manager.cpp b/slobrok/src/vespa/slobrok/server/exchange_manager.cpp index 94e951ca252..89167842c1b 100644 --- a/slobrok/src/vespa/slobrok/server/exchange_manager.cpp +++ b/slobrok/src/vespa/slobrok/server/exchange_manager.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "exchange_manager.h" -#include "rpc_server_map.h" #include "sbenv.h" #include <vespa/fnet/frt/supervisor.h> #include <vespa/vespalib/util/overload.h> @@ -14,11 +13,9 @@ namespace slobrok { //----------------------------------------------------------------------------- -ExchangeManager::ExchangeManager(SBEnv &env, RpcServerMap &rpcsrvmap) +ExchangeManager::ExchangeManager(SBEnv &env) : _partners(), - _env(env), - _rpcsrvmanager(env.rpcServerManager()), - _rpcsrvmap(rpcsrvmap) + _env(env) { } @@ -69,35 +66,13 @@ ExchangeManager::getPartnerList() void ExchangeManager::forwardRemove(const std::string & name, const std::string & spec) { - WorkPackage *package = new WorkPackage(WorkPackage::OP_REMOVE, *this, - ScriptCommand::makeIgnoreCmd(_env, name, spec)); + WorkPackage *package = new WorkPackage(WorkPackage::OP_REMOVE, ServiceMapping{name, spec}, *this); for (const auto & entry : _partners) { package->addItem(entry.second.get()); } package->expedite(); } -void -ExchangeManager::doAdd(ScriptCommand rdc) -{ - WorkPackage *package = new WorkPackage(WorkPackage::OP_DOADD, *this, std::move(rdc)); - - for (const auto & entry : _partners) { - package->addItem(entry.second.get()); - } - package->expedite(); -} - - -void -ExchangeManager::wantAdd(ScriptCommand rdc) -{ - WorkPackage *package = new WorkPackage(WorkPackage::OP_WANTADD, *this, std::move(rdc)); - for (const auto & entry : _partners) { - package->addItem(entry.second.get()); - } - package->expedite(); -} RemoteSlobrok * ExchangeManager::lookupPartner(const std::string & name) const { @@ -128,22 +103,8 @@ void ExchangeManager::healthCheck() { auto newWorldList = env().consensusMap().currentConsensus(); - if (! _env.useNewLogic()) { - auto oldWorldServices = env().rpcServerMap().allManaged(); - ServiceMappingList oldWorldList; - for (const auto *nsp : oldWorldServices) { - oldWorldList.emplace_back(nsp->getName(), nsp->getSpec()); - } - std::sort(oldWorldList.begin(), oldWorldList.end()); - vespalib::string diff = diffLists(oldWorldList, newWorldList); - if (! diff.empty()) { - LOG(warning, "Diff from old world rpcServerMap to new world consensus map: %s", - diff.c_str()); - } - } for (const auto & [ name, partner ] : _partners) { partner->maybeStartFetch(); - partner->maybePushMine(); auto remoteList = partner->remoteMap().allMappings(); // 0 is expected (when remote is down) if (remoteList.size() != 0) { @@ -205,14 +166,12 @@ ExchangeManager::WorkPackage::WorkItem::~WorkItem() } -ExchangeManager::WorkPackage::WorkPackage(op_type op, - ExchangeManager &exchanger, - ScriptCommand script) +ExchangeManager::WorkPackage::WorkPackage(op_type op, const ServiceMapping &mapping, ExchangeManager &exchanger) : _work(), _doneCnt(0), _numDenied(0), - _script(std::move(script)), _exchanger(exchanger), + _mapping(mapping), _optype(op) { } @@ -230,9 +189,9 @@ ExchangeManager::WorkPackage::doneItem(bool denied) (int)_doneCnt, (int)_work.size(), (int)_numDenied); if (_doneCnt == _work.size()) { if (_numDenied > 0) { - _script.doneHandler(OkState(_numDenied, "denied by remote")); - } else { - _script.doneHandler(OkState()); + LOG(debug, "work package [%s->%s]: %zd/%zd denied by remote", + _mapping.name.c_str(), _mapping.spec.c_str(), + _numDenied, _doneCnt); } delete this; } @@ -245,18 +204,12 @@ ExchangeManager::WorkPackage::addItem(RemoteSlobrok *partner) if (! partner->isConnected()) { return; } - const char *name_p = _script.name().c_str(); - const char *spec_p = _script.spec().c_str(); + const char *name_p = _mapping.name.c_str(); + const char *spec_p = _mapping.spec.c_str(); FRT_RPCRequest *r = _exchanger._env.getSupervisor()->AllocRPCRequest(); - // XXX should recheck rpcsrvmap again - if (_optype == OP_REMOVE) { - r->SetMethodName("slobrok.internal.doRemove"); - } else if (_optype == OP_WANTADD) { - r->SetMethodName("slobrok.internal.wantAdd"); - } else if (_optype == OP_DOADD) { - r->SetMethodName("slobrok.internal.doAdd"); - } + LOG_ASSERT(_optype == OP_REMOVE); + r->SetMethodName("slobrok.internal.doRemove"); r->GetParams()->AddString(_exchanger._env.mySpec().c_str()); r->GetParams()->AddString(name_p); r->GetParams()->AddString(spec_p); @@ -274,7 +227,6 @@ ExchangeManager::WorkPackage::expedite() size_t sz = _work.size(); if (sz == 0) { // no remotes need doing. - _script.doneHandler(OkState()); delete this; return; } diff --git a/slobrok/src/vespa/slobrok/server/exchange_manager.h b/slobrok/src/vespa/slobrok/server/exchange_manager.h index 18a20274a75..6891686c94d 100644 --- a/slobrok/src/vespa/slobrok/server/exchange_manager.h +++ b/slobrok/src/vespa/slobrok/server/exchange_manager.h @@ -2,7 +2,6 @@ #pragma once #include "ok_state.h" -#include "cmd.h" #include "remote_slobrok.h" #include <deque> @@ -14,8 +13,6 @@ namespace slobrok { //----------------------------------------------------------------------------- class SBEnv; -class RpcServerMap; -class RpcServerManager; //----------------------------------------------------------------------------- @@ -61,37 +58,31 @@ private: std::vector<std::unique_ptr<WorkItem>> _work; size_t _doneCnt; size_t _numDenied; - ScriptCommand _script; public: ExchangeManager &_exchanger; - enum op_type { OP_NOP, OP_WANTADD, OP_DOADD, OP_REMOVE }; - op_type _optype; + enum op_type { OP_REMOVE }; + const ServiceMapping _mapping; + const op_type _optype; void addItem(RemoteSlobrok *partner); void doneItem(bool denied); void expedite(); WorkPackage(const WorkPackage&) = delete; WorkPackage& operator= (const WorkPackage&) = delete; - WorkPackage(op_type op, - ExchangeManager &exchanger, - ScriptCommand donehandler); + WorkPackage(op_type op, const ServiceMapping &mapping, ExchangeManager &exchanger); ~WorkPackage(); }; SBEnv &_env; - RpcServerManager &_rpcsrvmanager; - RpcServerMap &_rpcsrvmap; vespalib::string diffLists(const ServiceMappingList &lhs, const ServiceMappingList &rhs); public: ExchangeManager(const ExchangeManager &) = delete; ExchangeManager &operator=(const ExchangeManager &) = delete; - ExchangeManager(SBEnv &env, RpcServerMap &rpcsrvmap); + ExchangeManager(SBEnv &env); ~ExchangeManager(); SBEnv &env() { return _env; } - RpcServerManager &rpcServerManager() { return _rpcsrvmanager; } - RpcServerMap &rpcServerMap() { return _rpcsrvmap; } OkState addPartner(const std::string & spec); void removePartner(const std::string & spec); @@ -99,9 +90,6 @@ public: void forwardRemove(const std::string & name, const std::string & spec); - void wantAdd(ScriptCommand rdc); - void doAdd(ScriptCommand rdc); - RemoteSlobrok *lookupPartner(const std::string & name) const; void healthCheck(); }; diff --git a/slobrok/src/vespa/slobrok/server/local_rpc_monitor_map.h b/slobrok/src/vespa/slobrok/server/local_rpc_monitor_map.h index 96c0ee03245..173a0455e43 100644 --- a/slobrok/src/vespa/slobrok/server/local_rpc_monitor_map.h +++ b/slobrok/src/vespa/slobrok/server/local_rpc_monitor_map.h @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "cmd.h" -#include "managed_rpc_server.h" #include "map_listener.h" #include "map_source.h" #include "mapping_monitor.h" @@ -12,6 +10,8 @@ #include "service_map_history.h" #include "service_mapping.h" +#include <vespa/fnet/task.h> + #include <vector> #include <memory> #include <map> diff --git a/slobrok/src/vespa/slobrok/server/remote_check.cpp b/slobrok/src/vespa/slobrok/server/remote_check.cpp index 157b959dbfe..da4d1ebc3dd 100644 --- a/slobrok/src/vespa/slobrok/server/remote_check.cpp +++ b/slobrok/src/vespa/slobrok/server/remote_check.cpp @@ -2,8 +2,6 @@ #include "remote_check.h" #include "named_service.h" -#include "rpc_server_map.h" -#include "rpc_server_manager.h" #include "remote_slobrok.h" #include "random.h" #include "exchange_manager.h" @@ -13,12 +11,9 @@ LOG_SETUP(".slobrok.server.remote_check"); namespace slobrok { -RemoteCheck::RemoteCheck(FNET_Scheduler *sched, - RpcServerMap& rpcsrvmap, - RpcServerManager& rpcsrvman, - ExchangeManager& exch) +RemoteCheck::RemoteCheck(FNET_Scheduler *sched, ExchangeManager& exch) : FNET_Task(sched), - _rpcsrvmap(rpcsrvmap), _rpcsrvmanager(rpcsrvman), _exchanger(exch) + _exchanger(exch) { double seconds = randomIn(15.3, 27.9); Schedule(seconds); diff --git a/slobrok/src/vespa/slobrok/server/remote_check.h b/slobrok/src/vespa/slobrok/server/remote_check.h index e0cf89c177d..11eb85401fe 100644 --- a/slobrok/src/vespa/slobrok/server/remote_check.h +++ b/slobrok/src/vespa/slobrok/server/remote_check.h @@ -20,17 +20,12 @@ class ExchangeManager; class RemoteCheck : public FNET_Task { private: - RpcServerMap &_rpcsrvmap; - RpcServerManager &_rpcsrvmanager; ExchangeManager &_exchanger; RemoteCheck(const RemoteCheck &); // Not used RemoteCheck &operator=(const RemoteCheck &); // Not used public: - explicit RemoteCheck(FNET_Scheduler *sched, - RpcServerMap& rpcsrvmap, - RpcServerManager& rpcsrvman, - ExchangeManager& exchanger); + explicit RemoteCheck(FNET_Scheduler *sched, ExchangeManager& exchanger); ~RemoteCheck(); private: void PerformTask() override; diff --git a/slobrok/src/vespa/slobrok/server/remote_slobrok.cpp b/slobrok/src/vespa/slobrok/server/remote_slobrok.cpp index d867d955dac..4b308e90e37 100644 --- a/slobrok/src/vespa/slobrok/server/remote_slobrok.cpp +++ b/slobrok/src/vespa/slobrok/server/remote_slobrok.cpp @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "remote_slobrok.h" -#include "rpc_server_map.h" -#include "rpc_server_manager.h" #include "exchange_manager.h" #include "sbenv.h" #include <vespa/fnet/frt/supervisor.h> @@ -18,7 +16,6 @@ namespace slobrok { RemoteSlobrok::RemoteSlobrok(const std::string &name, const std::string &spec, ExchangeManager &manager) : _exchanger(manager), - _rpcsrvmanager(manager.rpcServerManager()), _remote(nullptr), _serviceMapMirror(), _rpcserver(name, spec, *this), @@ -26,11 +23,7 @@ RemoteSlobrok::RemoteSlobrok(const std::string &name, const std::string &spec, _failCnt(0), _consensusSubscription(MapSubscription::subscribe(_serviceMapMirror, _exchanger.env().consensusMap())), _remAddPeerReq(nullptr), - _remListReq(nullptr), - _remAddReq(nullptr), - _remRemReq(nullptr), - _remFetchReq(nullptr), - _pending() + _remFetchReq(nullptr) { _rpcserver.healthCheck(); } @@ -38,8 +31,6 @@ RemoteSlobrok::RemoteSlobrok(const std::string &name, const std::string &spec, void RemoteSlobrok::shutdown() { _reconnecter.disable(); - _pending.clear(); - if (_remote != nullptr) { _remote->SubRef(); _remote = nullptr; @@ -51,15 +42,6 @@ void RemoteSlobrok::shutdown() { if (_remAddPeerReq != nullptr) { _remAddPeerReq->Abort(); } - if (_remListReq != nullptr) { - _remListReq->Abort(); - } - if (_remAddReq != nullptr) { - _remAddReq->Abort(); - } - if (_remRemReq != nullptr) { - _remRemReq->Abort(); - } _serviceMapMirror.clear(); } @@ -68,39 +50,6 @@ RemoteSlobrok::~RemoteSlobrok() { // _rpcserver destructor called automatically } -void -RemoteSlobrok::doPending() -{ - if (_remAddReq != nullptr) return; - if (_remRemReq != nullptr) return; - - if (_remote == nullptr) return; - - if ( ! _pending.empty() ) { - std::unique_ptr<NamedService> todo = std::move(_pending.front()); - _pending.pop_front(); - - const NamedService *rpcsrv = _exchanger.rpcServerMap().lookup(todo->getName()); - - if (rpcsrv == nullptr) { - _remRemReq = getSupervisor()->AllocRPCRequest(); - _remRemReq->SetMethodName("slobrok.internal.doRemove"); - _remRemReq->GetParams()->AddString(_exchanger.env().mySpec().c_str()); - _remRemReq->GetParams()->AddString(todo->getName().c_str()); - _remRemReq->GetParams()->AddString(todo->getSpec().c_str()); - _remote->InvokeAsync(_remRemReq, 2.0, this); - } else { - _remAddReq = getSupervisor()->AllocRPCRequest(); - _remAddReq->SetMethodName("slobrok.internal.doAdd"); - _remAddReq->GetParams()->AddString(_exchanger.env().mySpec().c_str()); - _remAddReq->GetParams()->AddString(todo->getName().c_str()); - _remAddReq->GetParams()->AddString(rpcsrv->getSpec().c_str()); - _remote->InvokeAsync(_remAddReq, 2.0, this); - } - // XXX should save this and pick up on RequestDone() - } -} - void RemoteSlobrok::maybeStartFetch() { if (_remFetchReq != nullptr) return; if (_remote == nullptr) return; @@ -168,21 +117,6 @@ void RemoteSlobrok::handleFetchResult() { } } - - -void -RemoteSlobrok::pushMine() -{ - // all mine - std::vector<const NamedService *> mine = _exchanger.rpcServerMap().allManaged(); - while (mine.size() > 0) { - const NamedService *now = mine.back(); - mine.pop_back(); - _pending.push_back(std::make_unique<NamedService>(now->getName(), now->getSpec())); - } - doPending(); -} - void RemoteSlobrok::RequestDone(FRT_RPCRequest *req) { @@ -190,7 +124,6 @@ RemoteSlobrok::RequestDone(FRT_RPCRequest *req) handleFetchResult(); return; } - FRT_Values &answer = *(req->GetReturn()); if (req == _remAddPeerReq) { // handle response after asking remote slobrok to add me as a peer: if (req->IsError()) { @@ -201,96 +134,15 @@ RemoteSlobrok::RequestDone(FRT_RPCRequest *req) myname, myspec, getName().c_str(), getSpec().c_str(), req->GetErrorMessage()); req->SubRef(); _remAddPeerReq = nullptr; - goto retrylater; + fail(); + return; } req->SubRef(); _remAddPeerReq = nullptr; - // next step is to ask the remote to send its list of managed names: - LOG_ASSERT(_remListReq == nullptr); - _remListReq = getSupervisor()->AllocRPCRequest(); - _remListReq->SetMethodName("slobrok.internal.listManagedRpcServers"); - if (_remote != nullptr) { - _remote->InvokeAsync(_remListReq, 3.0, this); - } - // when _remListReq is returned, our managed list is added - } else if (req == _remListReq) { - // handle the list sent from the remote: - if (req->IsError() - || strcmp(answer.GetTypeString(), "SS") != 0) - { - LOG(error, "error listing remote slobrok %s at %s: %s", - getName().c_str(), getSpec().c_str(), req->GetErrorMessage()); - req->SubRef(); - _remListReq = nullptr; - goto retrylater; - } - uint32_t numNames = answer.GetValue(0)._string_array._len; - uint32_t numSpecs = answer.GetValue(1)._string_array._len; - - if (numNames != numSpecs) { - LOG(error, "inconsistent array lengths from %s at %s", getName().c_str(), getSpec().c_str()); - req->SubRef(); - _remListReq = nullptr; - goto retrylater; - } - FRT_StringValue *names = answer.GetValue(0)._string_array._pt; - FRT_StringValue *specs = answer.GetValue(1)._string_array._pt; - - for (uint32_t idx = 0; idx < numNames; idx++) { - _rpcsrvmanager.addRemote(names[idx]._str, specs[idx]._str); - } - req->SubRef(); - _remListReq = nullptr; - - // next step is to push the ones I own: - maybeStartFetch(); - maybePushMine(); - } else if (req == _remAddReq) { - // handle response after pushing some name that we managed: - if (req->IsError() && (req->GetErrorCode() == FRTE_RPC_CONNECTION || - req->GetErrorCode() == FRTE_RPC_TIMEOUT)) - { - LOG(error, "connection error adding to remote slobrok: %s", req->GetErrorMessage()); - req->SubRef(); - _remAddReq = nullptr; - goto retrylater; - } - if (req->IsError()) { - FRT_Values &args = *req->GetParams(); - const char *rpcsrvname = args[1]._string._str; - const char *rpcsrvspec = args[2]._string._str; - LOG(warning, "error adding [%s -> %s] to remote slobrok: %s", - rpcsrvname, rpcsrvspec, req->GetErrorMessage()); - _rpcsrvmanager.removeLocal(rpcsrvname, rpcsrvspec); - } - req->SubRef(); - _remAddReq = nullptr; - doPending(); - } else if (req == _remRemReq) { - // handle response after pushing some remove we had pending: - if (req->IsError() && (req->GetErrorCode() == FRTE_RPC_CONNECTION || - req->GetErrorCode() == FRTE_RPC_TIMEOUT)) - { - LOG(error, "connection error adding to remote slobrok: %s", req->GetErrorMessage()); - req->SubRef(); - _remRemReq = nullptr; - goto retrylater; - } - if (req->IsError()) { - LOG(warning, "error removing on remote slobrok: %s", req->GetErrorMessage()); - } - req->SubRef(); - _remRemReq = nullptr; - doPending(); } else { LOG(error, "got unknown request back in RequestDone()"); LOG_ASSERT(req == nullptr); } - - return; - retrylater: - fail(); - return; } @@ -321,25 +173,6 @@ RemoteSlobrok::fail() _reconnecter.scheduleTryConnect(); } - -void -RemoteSlobrok::maybePushMine() -{ - if (_remote != nullptr && - _remAddPeerReq == nullptr && - _remListReq == nullptr && - _remAddReq == nullptr && - _remRemReq == nullptr) - { - LOG(debug, "spamming remote at %s with my names", getName().c_str()); - pushMine(); - } else { - LOG(debug, "not pushing mine, as we have: remote %p r.a.p.r=%p r.l.r=%p r.a.r=%p r.r.r=%p", - _remote, _remAddPeerReq, _remListReq, _remAddReq, _remRemReq); - } -} - - void RemoteSlobrok::notifyOkRpcSrv(ManagedRpcServer *rpcsrv) { @@ -357,10 +190,7 @@ RemoteSlobrok::notifyOkRpcSrv(ManagedRpcServer *rpcsrv) _remote = getSupervisor()->GetTarget(getSpec().c_str()); maybeStartFetch(); - // at this point, we will do (in sequence): - // ask peer to connect to us too; - // ask peer for its list of managed rpcservers, adding to our database - // add our managed rpcserver on peer + // at this point, we will ask peer to connect to us too; // any failure will cause disconnect and retry. _remAddPeerReq = getSupervisor()->AllocRPCRequest(); @@ -368,7 +198,6 @@ RemoteSlobrok::notifyOkRpcSrv(ManagedRpcServer *rpcsrv) _remAddPeerReq->GetParams()->AddString(_exchanger.env().mySpec().c_str()); _remAddPeerReq->GetParams()->AddString(_exchanger.env().mySpec().c_str()); _remote->InvokeAsync(_remAddPeerReq, 3.0, this); - // when _remAddPeerReq is returned, our managed list is added via doAdd() } void diff --git a/slobrok/src/vespa/slobrok/server/remote_slobrok.h b/slobrok/src/vespa/slobrok/server/remote_slobrok.h index b980aa90de0..ef7f39c08ed 100644 --- a/slobrok/src/vespa/slobrok/server/remote_slobrok.h +++ b/slobrok/src/vespa/slobrok/server/remote_slobrok.h @@ -2,7 +2,6 @@ #pragma once #include "ok_state.h" -#include "cmd.h" #include "i_rpc_server_manager.h" #include "managed_rpc_server.h" #include "service_map_mirror.h" @@ -12,7 +11,6 @@ namespace slobrok { //----------------------------------------------------------------------------- -class RpcServerManager; class ExchangeManager; //----------------------------------------------------------------------------- @@ -43,7 +41,6 @@ private: }; ExchangeManager &_exchanger; - RpcServerManager &_rpcsrvmanager; FRT_Target *_remote; ServiceMapMirror _serviceMapMirror; ManagedRpcServer _rpcserver; @@ -53,14 +50,8 @@ private: std::unique_ptr<MapSubscription> _consensusSubscription; FRT_RPCRequest *_remAddPeerReq; - FRT_RPCRequest *_remListReq; - FRT_RPCRequest *_remAddReq; - FRT_RPCRequest *_remRemReq; FRT_RPCRequest *_remFetchReq; - std::deque<std::unique_ptr<NamedService>> _pending; - void pushMine(); - void doPending(); void handleFetchResult(); public: @@ -72,7 +63,6 @@ public: void fail(); bool isConnected() const { return (_remote != nullptr); } void tryConnect(); - void maybePushMine(); void maybeStartFetch(); void invokeAsync(FRT_RPCRequest *req, double timeout, FRT_IRequestWait *rwaiter); const std::string & getName() const { return _rpcserver.getName(); } diff --git a/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp b/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp deleted file mode 100644 index a9a748323f7..00000000000 --- a/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp +++ /dev/null @@ -1,315 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "rpc_server_manager.h" -#include "reserved_name.h" -#include "rpc_server_map.h" -#include "remote_slobrok.h" -#include "sbenv.h" -#include <vespa/vespalib/util/stringfmt.h> -#include <sstream> - -#include <vespa/log/log.h> -LOG_SETUP(".slobrok.server.rpc_server_manager"); - -using vespalib::make_string_short::fmt; - -namespace slobrok { - -RpcServerManager::RpcServerManager(SBEnv &sbenv) - : FNET_Task(sbenv.getScheduler()), - _rpcsrvmap(sbenv.rpcServerMap()), - _exchanger(sbenv.exchangeManager()), - _env(sbenv), - _addManageds(), - _deleteList() -{ -} - -static OkState -validateName(const std::string & rpcsrvname) -{ - const char *p = rpcsrvname.c_str(); - while (*p != '\0') { - // important: disallow '*' - if (strchr("+,-./:=@[]_{}~<>" - "0123456789" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz", *p) == nullptr) - { - std::ostringstream tmp; - tmp << "Illegal character '" << *p << "' ("; - tmp << (int)(*p)<< ") in rpcserver name"; - return OkState(13, tmp.str().c_str()); - } - ++p; - } - if (p == rpcsrvname) { - return OkState(13, "empty rpcserver name"); - } - return OkState(); -} - - -OkState -RpcServerManager::checkPartner(const std::string & remslobrok) -{ - if (remslobrok == _env.mySpec()) { - return OkState(13, "remote slobrok using my rpcserver name"); - } - const RemoteSlobrok *partner = _exchanger.lookupPartner(remslobrok); - if (partner == nullptr) { - return OkState(13, "remote slobrok not a partner"); - } - return OkState(); -} - -OkState -RpcServerManager::addRemReservation(const std::string & remslobrok, const std::string & name, const std::string &spec) -{ - OkState state = checkPartner(remslobrok); - if (state.failed()) return state; - - OkState valid = validateName(name); - if (valid.failed()) return valid; - - const NamedService *old = _rpcsrvmap.lookupManaged(name); - if (old != nullptr) { - if (old->getSpec() == spec) { - // was alright already - return OkState(0, "already registered"); - } - LOG(warning, "remote %s tried to register [%s -> %s] but we already have [%s -> %s] registered!", - remslobrok.c_str(), name.c_str(), spec.c_str(), old->getName().c_str(), old->getSpec().c_str()); - return OkState(FRTE_RPC_METHOD_FAILED, "already managed by me"); - } - if (_rpcsrvmap.conflictingReservation(name, spec)) { - return OkState(FRTE_RPC_METHOD_FAILED, "registration for name already in progress"); - } - _rpcsrvmap.addReservation(std::make_unique<ReservedName>(name, spec, false)); - return OkState(0, "done"); -} - - -OkState -RpcServerManager::addMyReservation(const std::string & name, const std::string & spec) -{ - OkState valid = validateName(name); - if (valid.failed()) return valid; - - const NamedService *old = _rpcsrvmap.lookupManaged(name); - if (old != nullptr) { - if (old->getSpec() == spec) { - // was alright already - return OkState(0, "already registered"); - } else { - return OkState(FRTE_RPC_METHOD_FAILED, fmt("name %s registered (to %s), cannot register %s", - name.c_str(), old->getSpec().c_str(), spec.c_str())); - } - } - - // check if we already are in the progress of adding this - if (_rpcsrvmap.conflictingReservation(name, spec)) { - const ReservedName * rsv = _rpcsrvmap.getReservation(name); - LOG(warning, "conflicting registrations: wanted [%s -> %s] but [%s -> %s] already reserved", - name.c_str(), spec.c_str(), rsv->getName().c_str(), rsv->getSpec().c_str()); - return OkState(FRTE_RPC_METHOD_FAILED, - "registration for name already in progress with a different spec"); - } - _rpcsrvmap.removeReservation(name); - _rpcsrvmap.addReservation(std::make_unique<ReservedName>(name, spec, true)); - return OkState(0, "done"); -} - - -OkState -RpcServerManager::addRemote(const std::string & name, const std::string &spec) -{ - OkState valid = validateName(name); - if (valid.failed()) return valid; - - if (alreadyManaged(name, spec)) { - return OkState(0, "already correct"); - } - const NamedService *old = _rpcsrvmap.lookup(name); - if (old != nullptr) { - if (old->getSpec() != spec) { - LOG(warning, "collision on remote add: name %s registered to %s locally, " - "but another location broker wants it registered to %s", - name.c_str(), old->getSpec().c_str(), spec.c_str()); - removeRemote(name, old->getSpec()); - return OkState(13, "registered, with different spec"); - } - // was alright already, remove reservation - _rpcsrvmap.removeReservation(name); - return OkState(0, "already correct"); - } - _rpcsrvmap.removeReservation(name); - auto rpcsrv = std::make_unique<ManagedRpcServer>(name, spec, *this); - _rpcsrvmap.addNew(std::move(rpcsrv)); - return OkState(0, "done"); -} - -OkState -RpcServerManager::remove(ManagedRpcServer *rpcsrv) -{ - const NamedService *td = _rpcsrvmap.lookup(rpcsrv->getName()); - if (td == rpcsrv) { - return removeLocal(rpcsrv->getName(), rpcsrv->getSpec()); - } else { - return OkState(1, "not currently registered"); - } -} - - -OkState -RpcServerManager::removeRemote(const std::string &name, const std::string &spec) -{ - const NamedService *old = _rpcsrvmap.lookup(name); - if (old == nullptr) { - // was alright already, remove any reservation too - _rpcsrvmap.removeReservation(name); - return OkState(0, "already done"); - } - if (old->getSpec() != spec) { - return OkState(1, "name registered, but with different spec"); - } - std::unique_ptr<NamedService> td = _rpcsrvmap.remove(name); - LOG_ASSERT(td.get() == old); - return OkState(0, "done"); -} - -OkState -RpcServerManager::removeLocal(const std::string & name, const std::string &spec) -{ - const NamedService *td = _rpcsrvmap.lookup(name); - if (td == nullptr) { - // already removed, nop - return OkState(); - } - - const RemoteSlobrok *partner = _exchanger.lookupPartner(name); - if (partner != nullptr) { - return OkState(13, "cannot unregister partner slobrok"); - } - - const ManagedRpcServer *rpcsrv = _rpcsrvmap.lookupManaged(name); - if (rpcsrv == nullptr) { - return OkState(13, "not a local rpcserver"); - } - - if (rpcsrv->getSpec() != spec) { - // the client can probably ignore this "error" - // or log it on level INFO? - return OkState(1, fmt("name registered, but with different spec (%s)", rpcsrv->getSpec().c_str())); - } - auto tdUP = _rpcsrvmap.remove(name); - LOG_ASSERT(tdUP.get() == rpcsrv); - _exchanger.forwardRemove(name, spec); - return OkState(); -} - - -void -RpcServerManager::addManaged(ScriptCommand rdc) -{ - const std::string &name = rdc.name(); - const std::string &spec = rdc.spec(); - auto newRpcServer = std::make_unique<ManagedRpcServer>(name, spec, *this); - ManagedRpcServer & rpcsrv = *newRpcServer; - _rpcsrvmap.addNew(std::move(newRpcServer)); - for (size_t i = 0; i < _addManageds.size(); i++) { - if (_addManageds[i].rpcsrv == nullptr) { - _addManageds[i].rpcsrv = &rpcsrv; - _addManageds[i].handler = std::move(rdc); - rpcsrv.healthCheck(); - return; - } - } - _addManageds.emplace_back(&rpcsrv, std::move(rdc)); - rpcsrv.healthCheck(); - return; -} - - - -bool -RpcServerManager::alreadyManaged(const std::string &name, const std::string &spec) -{ - const ManagedRpcServer *rpcsrv = _rpcsrvmap.lookupManaged(name); - if (rpcsrv != nullptr) { - if (rpcsrv->getSpec() == spec) { - return true; - } - } - return false; -} - - -RpcServerManager::~RpcServerManager() -{ - Kill(); - PerformTask(); -} - - -void -RpcServerManager::PerformTask() -{ - std::vector<std::unique_ptr<NamedService>> deleteAfterSwap; - std::swap(deleteAfterSwap, _deleteList); -} - - -void -RpcServerManager::notifyFailedRpcSrv(ManagedRpcServer *rpcsrv, std::string errmsg) -{ - _env.countFailedHeartbeat(); - const auto &name = rpcsrv->getName(); - const auto &spec = rpcsrv->getSpec(); - const char *namep = name.c_str(); - const char *specp = spec.c_str(); - std::unique_ptr<NamedService> toDelete; - const NamedService *old = _rpcsrvmap.lookup(rpcsrv->getName()); - if (old == rpcsrv) { - toDelete = _rpcsrvmap.remove(name); - LOG_ASSERT(toDelete.get() == rpcsrv); - LOG(info, "managed server %s at %s failed: %s", namep, specp, errmsg.c_str()); - } else { - // only managed servers should exist, this is bad: - LOG(error, "unmanaged server %s at %s failed: %s", namep, specp, errmsg.c_str()); - } - _exchanger.forwardRemove(name, spec); - for (size_t i = 0; i < _addManageds.size(); ++i) { - if (_addManageds[i].rpcsrv == rpcsrv) { - LOG(warning, "rpcserver %s at %s failed while trying to register", namep, specp); - _addManageds[i].rpcsrv = nullptr; - _addManageds[i].handler.doneHandler(OkState(13, "failed check using listNames callback")); - } - } - if (toDelete) { - _deleteList.push_back(std::move(toDelete)); - ScheduleNow(); - } -} - -void -RpcServerManager::notifyOkRpcSrv(ManagedRpcServer *rpcsrv) -{ - for (size_t i = 0; i < _addManageds.size(); ++i) { - if (_addManageds[i].rpcsrv == rpcsrv) { - _addManageds[i].handler.doneHandler(OkState()); - _addManageds[i].rpcsrv = 0; - } - } - // XXX check if pending wantAdd / doAdd / registerRpcServer -} - -FRT_Supervisor * -RpcServerManager::getSupervisor() -{ - return _env.getSupervisor(); -} - -//----------------------------------------------------------------------------- - -} // namespace slobrok diff --git a/slobrok/src/vespa/slobrok/server/rpc_server_manager.h b/slobrok/src/vespa/slobrok/server/rpc_server_manager.h deleted file mode 100644 index 15b674388e3..00000000000 --- a/slobrok/src/vespa/slobrok/server/rpc_server_manager.h +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "ok_state.h" -#include "cmd.h" -#include "i_rpc_server_manager.h" -#include "named_service.h" -#include <vespa/fnet/task.h> -#include <vector> -#include <memory> - -namespace slobrok { - -class NamedService; -class ManagedRpcServer; -class RemoteSlobrok; -class ReservedName; -class RpcServerMap; -class ExchangeManager; -class SBEnv; - -/** - * @class RpcServerManager - * @brief Main "business logic" for the service location broker. - * - * Used by all external and some internal operations. - * This class actually implements operations, - * checking for validity, manipulating internal datastructures, - * and initiating synchronization operations to peer slobroks. - **/ -class RpcServerManager : public FNET_Task, - public IRpcServerManager -{ -private: - RpcServerMap &_rpcsrvmap; - ExchangeManager &_exchanger; - SBEnv &_env; - - struct MRSandRRSC { - ManagedRpcServer *rpcsrv; - ScriptCommand handler; - MRSandRRSC(ManagedRpcServer *d, ScriptCommand h) - : rpcsrv(d), handler(std::move(h)) {} - }; - std::vector<MRSandRRSC> _addManageds; - std::vector<std::unique_ptr<NamedService>> _deleteList; -public: - OkState checkPartner(const std::string & remslobrok); - - OkState addRemote(const std::string & name, const std::string & spec); - - OkState addRemReservation(const std::string & remslobrok, const std::string & name, const std::string & spec); - OkState addMyReservation(const std::string & name, const std::string & spec); - - bool alreadyManaged(const std::string & name, const std::string & spec); - void addManaged(ScriptCommand rdc); - - OkState remove(ManagedRpcServer *rpcsrv); - - OkState removeLocal(const std::string & name, const std::string & spec); - OkState removeRemote(const std::string & name, const std::string & spec); - - RpcServerManager(const RpcServerManager &) = delete; - RpcServerManager &operator=(const RpcServerManager &) = delete; - RpcServerManager(SBEnv &sbenv); - ~RpcServerManager(); - - void PerformTask() override; - void notifyFailedRpcSrv(ManagedRpcServer *rpcsrv, std::string errmsg) override; - void notifyOkRpcSrv(ManagedRpcServer *rpcsrv) override; - FRT_Supervisor *getSupervisor() override; -}; - -//----------------------------------------------------------------------------- - -} // namespace slobrok - diff --git a/slobrok/src/vespa/slobrok/server/rpc_server_map.cpp b/slobrok/src/vespa/slobrok/server/rpc_server_map.cpp deleted file mode 100644 index fcaaf57570c..00000000000 --- a/slobrok/src/vespa/slobrok/server/rpc_server_map.cpp +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "rpc_server_map.h" -#include "reserved_name.h" -#include "rpc_server_manager.h" -#include "sbenv.h" - -#include <vespa/log/log.h> -LOG_SETUP(".slobrok.server.rpc_server_map"); - -namespace slobrok { - -//----------------------------------------------------------------------------- - -ManagedRpcServer * -RpcServerMap::lookupManaged(const std::string & name) const { - auto found = _myrpcsrv_map.find(name); - return (found == _myrpcsrv_map.end()) ? nullptr : found->second.get(); -} - -const NamedService * -RpcServerMap::lookup(const std::string & name) const -{ - return lookupManaged(name); -} - -std::unique_ptr<NamedService> -RpcServerMap::remove(const std::string & name) -{ - auto service = std::move(_myrpcsrv_map[name]); - auto spec = service->getSpec(); - _proxy.remove(ServiceMapping{name, spec}); - _myrpcsrv_map.erase(name); - return service; -} - -std::vector<const NamedService *> -RpcServerMap::allManaged() const -{ - std::vector<const NamedService *> retval; - // get list of all names in myrpcsrv_map - for (const auto & entry : _myrpcsrv_map) { - retval.push_back(entry.second.get()); - } - return retval; -} - - -void -RpcServerMap::add(NamedService *rpcsrv) -{ - const std::string &name = rpcsrv->getName(); - - LOG_ASSERT(rpcsrv != nullptr); - LOG_ASSERT(_myrpcsrv_map.find(name) == _myrpcsrv_map.end()); - - removeReservation(name); - _proxy.add(ServiceMapping{name, rpcsrv->getSpec()}); -} - -void -RpcServerMap::addNew(std::unique_ptr<ManagedRpcServer> rpcsrv) -{ - const std::string &name = rpcsrv->getName(); - - auto oldman = std::move(_myrpcsrv_map[name]); - _myrpcsrv_map.erase(name); - - if (oldman) { - const ReservedName *oldres = _reservations[name].get(); - const std::string &spec = rpcsrv->getSpec(); - _proxy.remove(ServiceMapping{name, spec}); - const std::string &oldname = oldman->getName(); - const std::string &oldspec = oldman->getSpec(); - if (spec != oldspec) { - LOG(warning, "internal state problem: adding [%s at %s] but already had [%s at %s]", - name.c_str(), spec.c_str(), oldname.c_str(), oldspec.c_str()); - if (oldres != nullptr) { - const std::string &n = oldres->getName(); - const std::string &s = oldres->getSpec(); - LOG(warning, "old reservation: [%s at %s]", n.c_str(), s.c_str()); - } - } - } - add(rpcsrv.get()); - _myrpcsrv_map[name] = std::move(rpcsrv); -} - - -void -RpcServerMap::addReservation(std::unique_ptr<ReservedName> rpcsrv) -{ - LOG_ASSERT(rpcsrv != nullptr); - LOG_ASSERT(_myrpcsrv_map.find(rpcsrv->getName()) == _myrpcsrv_map.end()); - - // must not be reserved for something else already - // this should have been checked already, so assert - LOG_ASSERT(! conflictingReservation(rpcsrv->getName(), rpcsrv->getSpec())); - auto old = std::move(_reservations[rpcsrv->getName()]); - LOG_ASSERT(!old - || old->getSpec() == rpcsrv->getSpec() - || ! old->stillReserved()); - _reservations[rpcsrv->getName()] = std::move(rpcsrv); -} - - -/** check if there is a (different) registration for this name in progress */ -bool -RpcServerMap::conflictingReservation(const std::string &name, const std::string &spec) -{ - const ReservedName *resv = _reservations[name].get(); - return (resv != nullptr && - resv->stillReserved() && - resv->getSpec() != spec); -} - -const ReservedName * -RpcServerMap::getReservation(const std::string &name) const { - auto found = _reservations.find(name); - return (found == _reservations.end()) ? nullptr : found->second.get(); -} - -RpcServerMap::RpcServerMap() = default; - -RpcServerMap::~RpcServerMap() = default; - -void -RpcServerMap::removeReservation(const std::string & name) -{ - _reservations.erase(name); -} - -} // namespace slobrok diff --git a/slobrok/src/vespa/slobrok/server/rpc_server_map.h b/slobrok/src/vespa/slobrok/server/rpc_server_map.h deleted file mode 100644 index 3d2999069ea..00000000000 --- a/slobrok/src/vespa/slobrok/server/rpc_server_map.h +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "named_service.h" -#include "service_map_history.h" -#include "proxy_map_source.h" - -#include <memory> -#include <string> -#include <unordered_map> - -namespace slobrok { - -class NamedService; -class ManagedRpcServer; -class ReservedName; - -/** - * @class RpcServerMap - * @brief Contains the actual collections of NamedService (and subclasses) - * objects known by this location broker. - * - * Works as a collection of NamedService objects, but actually contains - * three seperate hashmaps. - **/ - -class RpcServerMap -{ -private: - using ManagedRpcServerMap = std::unordered_map<std::string, std::unique_ptr<ManagedRpcServer>>; - using ReservedNameMap = std::unordered_map<std::string, std::unique_ptr<ReservedName>>; - ManagedRpcServerMap _myrpcsrv_map; - ReservedNameMap _reservations; - ProxyMapSource _proxy; - - void add(NamedService *rpcsrv); - -public: - typedef std::vector<const NamedService *> RpcSrvlist; - - MapSource &proxy() { return _proxy; } - - ManagedRpcServer *lookupManaged(const std::string & name) const; - - const NamedService * lookup(const std::string & name) const; - RpcSrvlist allManaged() const; - - void addNew(std::unique_ptr<ManagedRpcServer> rpcsrv); - std::unique_ptr<NamedService> remove(const std::string & name); - - void addReservation(std::unique_ptr<ReservedName>rpcsrv); - bool conflictingReservation(const std::string & name, const std::string &spec); - - const ReservedName *getReservation(const std::string & name) const; - void removeReservation(const std::string & name); - - RpcServerMap(const RpcServerMap &) = delete; - RpcServerMap &operator=(const RpcServerMap &) = delete; - RpcServerMap(); - ~RpcServerMap(); -}; - -//----------------------------------------------------------------------------- - -} // namespace slobrok - diff --git a/slobrok/src/vespa/slobrok/server/rpchooks.cpp b/slobrok/src/vespa/slobrok/server/rpchooks.cpp index 540060210ed..6ce24be9201 100644 --- a/slobrok/src/vespa/slobrok/server/rpchooks.cpp +++ b/slobrok/src/vespa/slobrok/server/rpchooks.cpp @@ -4,8 +4,6 @@ #include "ok_state.h" #include "named_service.h" #include "request_completion_handler.h" -#include "rpc_server_map.h" -#include "rpc_server_manager.h" #include "remote_slobrok.h" #include "sbenv.h" #include "rpcmirror.h" @@ -38,12 +36,31 @@ public: ~MetricsReport() override { Kill(); } }; +bool match(const char *name, const char *pattern) { + LOG_ASSERT(name != nullptr); + LOG_ASSERT(pattern != nullptr); + while (*pattern != '\0') { + if (*name == *pattern) { + ++name; + ++pattern; + } else if (*pattern == '*') { + ++pattern; + while (*name != '/' && *name != '\0') { + ++name; + } + } else { + return false; + } + } + return (*name == *pattern); +} + } // namespace <unnamed> //----------------------------------------------------------------------------- -RPCHooks::RPCHooks(SBEnv &env, RpcServerMap& rpcsrvmap, RpcServerManager& rpcsrvman) - : _env(env), _rpcsrvmap(rpcsrvmap), _rpcsrvmanager(rpcsrvman), +RPCHooks::RPCHooks(SBEnv &env) + : _env(env), _globalHistory(env.globalHistory()), _localHistory(env.localHistory()), _cnts(Metrics::zero()), @@ -65,25 +82,6 @@ void RPCHooks::reportMetrics() { EV_COUNT("other_reqs", _cnts.otherReqs); } -bool RPCHooks::match(const char *name, const char *pattern) { - LOG_ASSERT(name != nullptr); - LOG_ASSERT(pattern != nullptr); - while (*pattern != '\0') { - if (*name == *pattern) { - ++name; - ++pattern; - } else if (*pattern == '*') { - ++pattern; - while (*name != '/' && *name != '\0') { - ++name; - } - } else { - return false; - } - } - return (*name == *pattern); -} - void RPCHooks::initRPC(FRT_Supervisor *supervisor) { _m_reporter = std::make_unique<MetricsReport>(supervisor, *this); @@ -229,11 +227,6 @@ void RPCHooks::initRPC(FRT_Supervisor *supervisor) { //------------------------------------------------------------------------- } - -bool RPCHooks::useNewLogic() const { - return _env.useNewLogic(); -} - void RPCHooks::rpc_listNamesServed(FRT_RPCRequest *req) { FRT_Values &dst = *req->GetReturn(); FRT_StringValue *names = dst.AddStringArray(1); @@ -257,9 +250,6 @@ void RPCHooks::rpc_registerRpcServer(FRT_RPCRequest *req) { } req->Detach(); _env.localMonitorMap().addLocal(mapping, std::make_unique<RequestCompletionHandler>(req)); - // TODO: remove this - auto script = ScriptCommand::makeRegRpcSrvCmd(_env, dName, dSpec, nullptr); - script.doRequest(); return; } diff --git a/slobrok/src/vespa/slobrok/server/rpchooks.h b/slobrok/src/vespa/slobrok/server/rpchooks.h index b68eb9007a8..bd051df64f1 100644 --- a/slobrok/src/vespa/slobrok/server/rpchooks.h +++ b/slobrok/src/vespa/slobrok/server/rpchooks.h @@ -40,8 +40,6 @@ public: private: SBEnv &_env; - RpcServerMap &_rpcsrvmap; - RpcServerManager &_rpcsrvmanager; ServiceMapHistory &_globalHistory; ServiceMapHistory &_localHistory; @@ -49,44 +47,33 @@ private: std::unique_ptr<FNET_Task> _m_reporter; public: - RPCHooks(SBEnv &env, RpcServerMap& rpcsrvmap, RpcServerManager& rpcsrvman); + RPCHooks(SBEnv &env); ~RPCHooks() override; - static bool match(const char *name, const char *pattern); - void initRPC(FRT_Supervisor *supervisor); void reportMetrics(); const Metrics& getMetrics() const { return _cnts; } void countFailedHeartbeat() { _cnts.heartBeatFails++; } private: - bool useNewLogic() const; - - void rpc_lookupRpcServer(FRT_RPCRequest *req); - - void new_registerRpcServer(FRT_RPCRequest *req); - void new_unregisterRpcServer(FRT_RPCRequest *req); - void new_wantAdd(FRT_RPCRequest *req); - void new_doRemove(FRT_RPCRequest *req); - void new_doAdd(FRT_RPCRequest *req); - void rpc_registerRpcServer(FRT_RPCRequest *req); void rpc_unregisterRpcServer(FRT_RPCRequest *req); - void rpc_addPeer(FRT_RPCRequest *req); void rpc_removePeer(FRT_RPCRequest *req); + void rpc_incrementalFetch(FRT_RPCRequest *req); + void rpc_doRemove(FRT_RPCRequest *req); + void rpc_fetchLocalView(FRT_RPCRequest *req); + void rpc_listNamesServed(FRT_RPCRequest *req); + + /** for unit tests and debugging, consider removing some of these: */ + void rpc_lookupRpcServer(FRT_RPCRequest *req); void rpc_listManagedRpcServers(FRT_RPCRequest *req); void rpc_lookupManaged(FRT_RPCRequest *req); void rpc_listAllRpcServers(FRT_RPCRequest *req); - void rpc_incrementalFetch(FRT_RPCRequest *req); void rpc_wantAdd(FRT_RPCRequest *req); void rpc_doAdd(FRT_RPCRequest *req); - void rpc_doRemove(FRT_RPCRequest *req); - void rpc_fetchLocalView(FRT_RPCRequest *req); - - void rpc_listNamesServed(FRT_RPCRequest *req); - void rpc_getRpcServerHistory(FRT_RPCRequest *req); + /** consider removing: */ void rpc_stop(FRT_RPCRequest *req); void rpc_version(FRT_RPCRequest *req); }; diff --git a/slobrok/src/vespa/slobrok/server/sbenv.cpp b/slobrok/src/vespa/slobrok/server/sbenv.cpp index 42debe1556c..1d279994cb0 100644 --- a/slobrok/src/vespa/slobrok/server/sbenv.cpp +++ b/slobrok/src/vespa/slobrok/server/sbenv.cpp @@ -97,19 +97,16 @@ ConfigTask::PerformTask() } // namespace slobrok::<unnamed> -SBEnv::SBEnv(const ConfigShim &shim) : SBEnv(shim, true) {} - -SBEnv::SBEnv(const ConfigShim &shim, bool) +SBEnv::SBEnv(const ConfigShim &shim) : _transport(std::make_unique<FNET_Transport>(TransportConfig().drop_empty_buffers(true))), _supervisor(std::make_unique<FRT_Supervisor>(_transport.get())), _configShim(shim), _configurator(shim.factory().create(*this)), _shuttingDown(false), - _useNewLogic(true), _partnerList(), _me(createSpec(_configShim.portNumber())), - _rpcHooks(*this, _rpcsrvmap, _rpcsrvmanager), - _remotechecktask(std::make_unique<RemoteCheck>(getSupervisor()->GetScheduler(), _rpcsrvmap, _rpcsrvmanager, _exchanger)), + _rpcHooks(*this), + _remotechecktask(std::make_unique<RemoteCheck>(getSupervisor()->GetScheduler(), _exchanger)), _health(), _metrics(_rpcHooks, *_transport), _components(), @@ -117,9 +114,7 @@ SBEnv::SBEnv(const ConfigShim &shim, bool) [this] (MappingMonitorOwner &owner) { return std::make_unique<RpcMappingMonitor>(*_supervisor, owner); }), - _rpcsrvmanager(*this), - _exchanger(*this, _rpcsrvmap), - _rpcsrvmap() + _exchanger(*this) { srandom(time(nullptr) ^ getpid()); // note: feedback loop between these two: @@ -201,7 +196,6 @@ SBEnv::MainLoop() return 0; } - void SBEnv::setup(const std::vector<std::string> &cfg) { @@ -272,7 +266,7 @@ SBEnv::removePeer(const std::string &name, const std::string &spec) if (partner == nullptr) { return OkState(0, "remote slobrok not a partner"); } - _exchanger.removePartner(name); + _exchanger.removePartner(spec); return OkState(0, "done"); } diff --git a/slobrok/src/vespa/slobrok/server/sbenv.h b/slobrok/src/vespa/slobrok/server/sbenv.h index c6fd8905131..09986e037a9 100644 --- a/slobrok/src/vespa/slobrok/server/sbenv.h +++ b/slobrok/src/vespa/slobrok/server/sbenv.h @@ -3,8 +3,6 @@ #include "named_service.h" #include "rpc_mapping_monitor.h" -#include "rpc_server_map.h" -#include "rpc_server_manager.h" #include "remote_slobrok.h" #include "exchange_manager.h" #include "configshim.h" @@ -44,7 +42,6 @@ private: ConfigShim _configShim; Configurator::UP _configurator; bool _shuttingDown; - const bool _useNewLogic; SBEnv(const SBEnv &); // Not used SBEnv &operator=(const SBEnv &); // Not used @@ -62,9 +59,7 @@ private: UnionServiceMap _consensusMap; ServiceMapHistory _globalVisibleHistory; - RpcServerManager _rpcsrvmanager; ExchangeManager _exchanger; - RpcServerMap _rpcsrvmap; std::unique_ptr<MapSubscription> _localMonitorSubscription; std::unique_ptr<MapSubscription> _consensusSubscription; @@ -72,7 +67,6 @@ private: public: explicit SBEnv(const ConfigShim &shim); - SBEnv(const ConfigShim &shim, bool useNewConsensusLogic); ~SBEnv(); FNET_Transport *getTransport() { return _transport.get(); } @@ -83,9 +77,7 @@ public: void suspend(); void resume(); - RpcServerManager& rpcServerManager() { return _rpcsrvmanager; } ExchangeManager& exchangeManager() { return _exchanger; } - RpcServerMap& rpcServerMap() { return _rpcsrvmap; } ServiceMapHistory& globalHistory() { return _globalVisibleHistory; @@ -107,12 +99,11 @@ public: bool isSuspended() const { return false; } bool isShuttingDown() const { return _shuttingDown; } - bool useNewLogic() const { return _useNewLogic; } int MainLoop(); - OkState addPeer(const std::string& name, const std::string &spec); - OkState removePeer(const std::string& name, const std::string &spec); + OkState addPeer(const std::string& name, const std::string& spec); + OkState removePeer(const std::string& name, const std::string& spec); void countFailedHeartbeat() { _rpcHooks.countFailedHeartbeat(); } }; diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java b/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java index ed7d30c476f..a28d8be8f57 100644 --- a/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java @@ -24,6 +24,7 @@ import java.util.stream.Collectors; * @author ollivir */ public class LocalFileDb implements FileAcquirer, FileRegistry { + private final Map<FileReference, File> fileReferenceToFile = new HashMap<>(); private final Path appPath; @@ -37,7 +38,7 @@ public class LocalFileDb implements FileAcquirer, FileRegistry { synchronized (this) { File file = fileReferenceToFile.get(reference); if (file == null) { - throw new RuntimeException("Invalid file reference " + reference); + return new File(reference.value()); // Downloaded file reference: Will (hopefully) be resolved client side } return file; } @@ -48,6 +49,7 @@ public class LocalFileDb implements FileAcquirer, FileRegistry { } /* FileRegistry overrides */ + @Override public FileReference addFile(String relativePath) { File file = appPath.resolve(relativePath).toFile(); if (!file.exists()) { diff --git a/standalone-container/src/main/sh/standalone-container.sh b/standalone-container/src/main/sh/standalone-container.sh index b30a7ad4cd0..e6cbb435b6f 100755 --- a/standalone-container/src/main/sh/standalone-container.sh +++ b/standalone-container/src/main/sh/standalone-container.sh @@ -180,7 +180,6 @@ StartCommand() { -Djdisc.config.file="$cfpfile" \ -Djdisc.export.packages= \ -Djdisc.cache.path="$bundlecachedir" \ - -Djdisc.debug.resources=false \ -Djdisc.bundle.path="$VESPA_HOME/lib/jars" \ -Djdisc.logger.enabled=true \ -Djdisc.logger.level=ALL \ diff --git a/storage/src/tests/distributor/distributor_stripe_test.cpp b/storage/src/tests/distributor/distributor_stripe_test.cpp index 418b224eb25..067b0efdd5c 100644 --- a/storage/src/tests/distributor/distributor_stripe_test.cpp +++ b/storage/src/tests/distributor/distributor_stripe_test.cpp @@ -230,6 +230,56 @@ TEST_F(DistributorStripeTest, operations_generated_and_started_without_duplicate ASSERT_EQ(6, _sender.commands().size()); } +TEST_F(DistributorStripeTest, maintenance_scheduling_inhibited_if_cluster_state_is_pending) +{ + setup_stripe(Redundancy(2), NodeCount(4), "storage:3 distributor:1"); + simulate_set_pending_cluster_state("storage:4 distributor:1"); + + _sender.commands().clear(); // Remove pending bucket info requests + + tickDistributorNTimes(1); + EXPECT_FALSE(stripe_is_in_recovery_mode()); + + for (uint32_t i = 0; i < 6; ++i) { + addNodesToBucketDB(document::BucketId(16, i), "0=2"); // Needs activation, merging + } + tickDistributorNTimes(10); + + // No ops should have been actually generated + ASSERT_EQ(0, _sender.commands().size()); +} + +TEST_F(DistributorStripeTest, non_activation_maintenance_inhibited_if_explicitly_toggled) +{ + setup_stripe(Redundancy(2), NodeCount(4), "storage:3 distributor:1"); + tickDistributorNTimes(1); + ASSERT_FALSE(stripe_is_in_recovery_mode()); + + for (uint32_t i = 0; i < 3; ++i) { + addNodesToBucketDB(document::BucketId(16, i), "0=2/3/4/t/a"); // Needs merging, but not activation (already active) + } + _stripe->inhibit_non_activation_maintenance_operations(true); + tickDistributorNTimes(10); + + // No ops should have been actually generated + ASSERT_EQ("", _sender.getCommands()); +} + +TEST_F(DistributorStripeTest, activation_maintenance_not_inhibited_even_if_explicitly_toggled) +{ + setup_stripe(Redundancy(2), NodeCount(4), "storage:3 distributor:1"); + tickDistributorNTimes(1); + ASSERT_FALSE(stripe_is_in_recovery_mode()); + + for (uint32_t i = 0; i < 3; ++i) { + addNodesToBucketDB(document::BucketId(16, i), "0=2/3/4"); // Needs activation and merging + } + _stripe->inhibit_non_activation_maintenance_operations(true); + tickDistributorNTimes(10); + + ASSERT_EQ("SetBucketState,SetBucketState,SetBucketState", _sender.getCommands()); +} + TEST_F(DistributorStripeTest, recovery_mode_on_cluster_state_change) { setup_stripe(Redundancy(1), NodeCount(2), @@ -425,7 +475,7 @@ TEST_F(DistributorStripeTest, merge_stats_are_accumulated_during_database_iterat // added to existing. tickDistributorNTimes(50); - const auto& stats = stripe_maintenance_stats(); + const auto stats = stripe_maintenance_stats(); { NodeMaintenanceStats wanted; wanted.syncing = 1; @@ -451,6 +501,11 @@ TEST_F(DistributorStripeTest, merge_stats_are_accumulated_during_database_iterat assertBucketSpaceStats(1, 3, 0, "default", bucketStats); assertBucketSpaceStats(0, 1, 1, "default", bucketStats); assertBucketSpaceStats(3, 1, 2, "default", bucketStats); + + EXPECT_EQ(stats.perNodeStats.total_replica_stats().movingOut, 1); + EXPECT_EQ(stats.perNodeStats.total_replica_stats().copyingOut, 2); + EXPECT_EQ(stats.perNodeStats.total_replica_stats().copyingIn, 2); + EXPECT_EQ(stats.perNodeStats.total_replica_stats().syncing, 2); } void @@ -484,7 +539,7 @@ TEST_F(DistributorStripeTest, stats_generated_for_preempted_operations) // by activation, we'll see no merge stats at all. addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1,1=2/2/2"); tickDistributorNTimes(50); - const auto& stats = stripe_maintenance_stats(); + const auto stats = stripe_maintenance_stats(); { NodeMaintenanceStats wanted; wanted.syncing = 1; diff --git a/storage/src/tests/distributor/top_level_distributor_test.cpp b/storage/src/tests/distributor/top_level_distributor_test.cpp index e1bd82ffb3e..9d01ce02f69 100644 --- a/storage/src/tests/distributor/top_level_distributor_test.cpp +++ b/storage/src/tests/distributor/top_level_distributor_test.cpp @@ -122,6 +122,12 @@ struct TopLevelDistributorTest : Test, TopLevelDistributorTestUtil { void assert_single_ok_remove_reply_present() { assert_single_reply_present_with_return_code(api::ReturnCode::OK); } + + void assert_all_stripes_are_maintenance_inhibited(bool inhibited) const { + for (auto* stripe : distributor_stripes()) { + EXPECT_EQ(stripe->non_activation_maintenance_is_inhibited(), inhibited); + } + } }; TopLevelDistributorTest::TopLevelDistributorTest() @@ -533,7 +539,7 @@ TEST_F(TopLevelDistributorTest, leaving_recovery_mode_immediately_sends_getnodes // TODO refactor this to set proper highest timestamp as part of bucket info // reply once we have the "highest timestamp across all owned buckets" feature // in place. -TEST_F(TopLevelDistributorTest, configured_safe_time_point_rejection_works_end_to_end) { +TEST_F(TopLevelDistributorTest, configured_feed_safe_time_point_rejection_works_end_to_end) { setup_distributor(Redundancy(2), NodeCount(2), "storage:1 distributor:2"); fake_clock().setAbsoluteTimeInSeconds(1000); @@ -558,4 +564,42 @@ TEST_F(TopLevelDistributorTest, configured_safe_time_point_rejection_works_end_t ASSERT_NO_FATAL_FAILURE(assert_single_ok_remove_reply_present()); } +TEST_F(TopLevelDistributorTest, configured_maintenance_safe_time_point_inhibition_works_end_to_end) { + setup_distributor(Redundancy(2), NodeCount(2), "storage:1 distributor:2"); + fake_clock().setAbsoluteTimeInSeconds(1000); + + auto cfg = current_distributor_config(); + cfg.maxClusterClockSkewSec = 10; + reconfigure(cfg); + + ASSERT_NO_FATAL_FAILURE(assert_all_stripes_are_maintenance_inhibited(false)); + + enable_distributor_cluster_state("storage:1 distributor:1", true); + tick_distributor_and_stripes_n_times(1); + ASSERT_NO_FATAL_FAILURE(assert_all_stripes_are_maintenance_inhibited(true)); + + fake_clock().setAbsoluteTimeInSeconds(1010); // Safe period still not expired + tick_distributor_and_stripes_n_times(1); + ASSERT_NO_FATAL_FAILURE(assert_all_stripes_are_maintenance_inhibited(true)); + + fake_clock().setAbsoluteTimeInSeconds(1011); // Safe period now expired + tick_distributor_and_stripes_n_times(1); + ASSERT_NO_FATAL_FAILURE(assert_all_stripes_are_maintenance_inhibited(false)); +} + +TEST_F(TopLevelDistributorTest, maintenance_safe_time_not_triggered_if_state_transition_does_not_have_ownership_transfer) { + setup_distributor(Redundancy(2), NodeCount(2), "storage:1 distributor:2"); + fake_clock().setAbsoluteTimeInSeconds(1000); + + auto cfg = current_distributor_config(); + cfg.maxClusterClockSkewSec = 10; + reconfigure(cfg); + + ASSERT_NO_FATAL_FAILURE(assert_all_stripes_are_maintenance_inhibited(false)); + + enable_distributor_cluster_state("storage:1 distributor:1", false); + tick_distributor_and_stripes_n_times(1); + ASSERT_NO_FATAL_FAILURE(assert_all_stripes_are_maintenance_inhibited(false)); +} + } diff --git a/storage/src/vespa/storage/config/distributorconfiguration.h b/storage/src/vespa/storage/config/distributorconfiguration.h index 7aa10893b80..09b30db086a 100644 --- a/storage/src/vespa/storage/config/distributorconfiguration.h +++ b/storage/src/vespa/storage/config/distributorconfiguration.h @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "config-stor-distributormanager.h" -#include "config-stor-visitordispatcher.h" +#include <vespa/storage/config/config-stor-distributormanager.h> +#include <vespa/storage/config/config-stor-visitordispatcher.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/storage/common/storagecomponent.h> #include <vespa/vespalib/util/time.h> diff --git a/storage/src/vespa/storage/distributor/cluster_state_bundle_activation_listener.h b/storage/src/vespa/storage/distributor/cluster_state_bundle_activation_listener.h index c3a885367ca..7bdd9b91bc1 100644 --- a/storage/src/vespa/storage/distributor/cluster_state_bundle_activation_listener.h +++ b/storage/src/vespa/storage/distributor/cluster_state_bundle_activation_listener.h @@ -17,7 +17,8 @@ namespace storage::distributor { class ClusterStateBundleActivationListener { public: virtual ~ClusterStateBundleActivationListener() = default; - virtual void on_cluster_state_bundle_activated(const lib::ClusterStateBundle&) = 0; + virtual void on_cluster_state_bundle_activated(const lib::ClusterStateBundle&, + bool has_bucket_ownership_transfer) = 0; }; } diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp b/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp index 9ec4d31eb32..3cfb8d70b7d 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp @@ -50,7 +50,7 @@ DistributorBucketSpace::enumerate_available_nodes() _distribution_bits = _clusterState->getDistributionBitCount(); auto node_count = _clusterState->getNodeCount(lib::NodeType::STORAGE); if (_pending_cluster_state) { - _distribution_bits = std::min(_distribution_bits, _pending_cluster_state->getDistributionBitCount()); + _distribution_bits = std::max(_distribution_bits, _pending_cluster_state->getDistributionBitCount()); node_count = std::min(node_count, _pending_cluster_state->getNodeCount(lib::NodeType::STORAGE)); } std::vector<bool> nodes(node_count); diff --git a/storage/src/vespa/storage/distributor/distributor_stripe.cpp b/storage/src/vespa/storage/distributor/distributor_stripe.cpp index 543264d97b9..da32b7ad4c6 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe.cpp +++ b/storage/src/vespa/storage/distributor/distributor_stripe.cpp @@ -75,8 +75,9 @@ DistributorStripe::DistributorStripe(DistributorComponentRegister& compReg, _db_memory_sample_interval(30s), _last_db_memory_sample_time_point(), _inhibited_maintenance_tick_count(0), - _must_send_updated_host_info(false), - _stripe_index(stripe_index) + _stripe_index(stripe_index), + _non_activation_maintenance_is_inhibited(false), + _must_send_updated_host_info(false) { propagateDefaultDistribution(_component.getDistribution()); propagateClusterStates(); @@ -557,8 +558,14 @@ DistributorStripe::propagateInternalScanMetricsToExternal() // All shared values are written when _metricLock is held, so no races. if (_bucketDBMetricUpdater.hasCompletedRound()) { - _bucketDbStats.propagateMetrics(_idealStateManager.getMetrics(), getMetrics()); - _idealStateManager.getMetrics().setPendingOperations(_maintenanceStats.global.pending); + auto& ideal_state_metrics = _idealStateManager.getMetrics(); + _bucketDbStats.propagateMetrics(ideal_state_metrics, getMetrics()); + ideal_state_metrics.setPendingOperations(_maintenanceStats.global.pending); + const auto& total_stats = _maintenanceStats.perNodeStats.total_replica_stats(); + ideal_state_metrics.buckets_replicas_moving_out.set(total_stats.movingOut); + ideal_state_metrics.buckets_replicas_copying_out.set(total_stats.copyingOut); + ideal_state_metrics.buckets_replicas_copying_in.set(total_stats.copyingIn); + ideal_state_metrics.buckets_replicas_syncing.set(total_stats.syncing); } } @@ -678,7 +685,11 @@ DistributorStripe::startNextMaintenanceOperation() { _throttlingStarter->setMaxPendingRange(getConfig().getMinPendingMaintenanceOps(), getConfig().getMaxPendingMaintenanceOps()); - _scheduler->tick(_schedulingMode); + auto effective_scheduling_mode = ((_schedulingMode == MaintenanceScheduler::RECOVERY_SCHEDULING_MODE) || + non_activation_maintenance_is_inhibited()) + ? MaintenanceScheduler::RECOVERY_SCHEDULING_MODE + : MaintenanceScheduler::NORMAL_SCHEDULING_MODE; + _scheduler->tick(effective_scheduling_mode); } framework::ThreadWaitInfo diff --git a/storage/src/vespa/storage/distributor/distributor_stripe.h b/storage/src/vespa/storage/distributor/distributor_stripe.h index 0dcac9ea7b7..59266b4cd61 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe.h +++ b/storage/src/vespa/storage/distributor/distributor_stripe.h @@ -22,6 +22,7 @@ #include <vespa/storageapi/message/state.h> #include <vespa/storageframework/generic/metric/metricupdatehook.h> #include <vespa/storageframework/generic/thread/tickingthread.h> +#include <atomic> #include <mutex> #include <queue> #include <unordered_map> @@ -182,6 +183,14 @@ public: return _db_memory_sample_interval; } + void inhibit_non_activation_maintenance_operations(bool inhibit) noexcept { + _non_activation_maintenance_is_inhibited.store(inhibit, std::memory_order_relaxed); + } + + bool non_activation_maintenance_is_inhibited() const noexcept { + return _non_activation_maintenance_is_inhibited.load(std::memory_order_relaxed); + } + bool tick() override; private: @@ -342,8 +351,9 @@ private: std::chrono::steady_clock::duration _db_memory_sample_interval; std::chrono::steady_clock::time_point _last_db_memory_sample_time_point; size_t _inhibited_maintenance_tick_count; - bool _must_send_updated_host_info; uint32_t _stripe_index; + std::atomic<bool> _non_activation_maintenance_is_inhibited; + bool _must_send_updated_host_info; }; } diff --git a/storage/src/vespa/storage/distributor/idealstatemanager.cpp b/storage/src/vespa/storage/distributor/idealstatemanager.cpp index 124f75ec169..f1d2b163623 100644 --- a/storage/src/vespa/storage/distributor/idealstatemanager.cpp +++ b/storage/src/vespa/storage/distributor/idealstatemanager.cpp @@ -21,8 +21,7 @@ using document::BucketSpace; using storage::lib::Node; using storage::lib::NodeType; -namespace storage { -namespace distributor { +namespace storage::distributor { IdealStateManager::IdealStateManager( const DistributorNodeContext& node_ctx, @@ -298,5 +297,4 @@ void IdealStateManager::getBucketStatus(std::ostream& out) const { } } -} // distributor -} // storage +} // storage::distributor diff --git a/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp b/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp index fd193ad6fd8..e786d81df91 100644 --- a/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp +++ b/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp @@ -86,7 +86,21 @@ IdealStateMetricSet::IdealStateMetricSet() {{"logdefault"},{"yamasdefault"}}, "The number of buckets that we are rechecking for " "ideal state operations", this), - startOperationsLatency("start_operations_latency", {}, "Time used in startOperations()", this), + buckets_replicas_moving_out("bucket_replicas_moving_out", + {{"logdefault"},{"yamasdefault"}}, + "Bucket replicas that should be moved out, e.g. retirement case or node " + "added to cluster that has higher ideal state priority.", this), + buckets_replicas_copying_in("bucket_replicas_copying_in", + {{"logdefault"},{"yamasdefault"}}, + "Bucket replicas that should be copied in, e.g. node does not have a " + "replica for a bucket that it is in ideal state for", this), + buckets_replicas_copying_out("bucket_replicas_copying_out", + {{"logdefault"},{"yamasdefault"}}, + "Bucket replicas that should be copied out, e.g. node is in ideal state " + "but might have to provide data other nodes in a merge", this), + buckets_replicas_syncing("bucket_replicas_syncing", + {{"logdefault"},{"yamasdefault"}}, + "Bucket replicas that need syncing due to mismatching metadata", this), nodesPerMerge("nodes_per_merge", {}, "The number of nodes involved in a single merge operation.", this) { createOperationMetrics(); diff --git a/storage/src/vespa/storage/distributor/idealstatemetricsset.h b/storage/src/vespa/storage/distributor/idealstatemetricsset.h index c1fb39bb50a..e9ccef9f93e 100644 --- a/storage/src/vespa/storage/distributor/idealstatemetricsset.h +++ b/storage/src/vespa/storage/distributor/idealstatemetricsset.h @@ -38,8 +38,11 @@ public: metrics::LongValueMetric buckets_toomanycopies; metrics::LongValueMetric buckets; metrics::LongValueMetric buckets_notrusted; - metrics::LongValueMetric buckets_rechecking; - metrics::LongAverageMetric startOperationsLatency; + metrics::LongValueMetric buckets_rechecking; // TODO remove, not used (but exposed by VespaMetricSet) + metrics::LongValueMetric buckets_replicas_moving_out; + metrics::LongValueMetric buckets_replicas_copying_in; + metrics::LongValueMetric buckets_replicas_copying_out; + metrics::LongValueMetric buckets_replicas_syncing; metrics::DoubleAverageMetric nodesPerMerge; void createOperationMetrics(); diff --git a/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.cpp b/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.cpp index 4e7f7d9d89d..db2eb6aadc9 100644 --- a/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.cpp +++ b/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.cpp @@ -34,9 +34,9 @@ merge_bucket_spaces_stats(NodeMaintenanceStatsTracker::BucketSpacesStats& dest, void NodeMaintenanceStatsTracker::merge(const NodeMaintenanceStatsTracker& rhs) { - for (const auto& entry : rhs._stats) { + for (const auto& entry : rhs._node_stats) { auto node_index = entry.first; - merge_bucket_spaces_stats(_stats[node_index], entry.second); + merge_bucket_spaces_stats(_node_stats[node_index], entry.second); } } diff --git a/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.h b/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.h index 6399e53089b..3c45bcdd5e5 100644 --- a/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.h +++ b/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.h @@ -50,7 +50,8 @@ public: using PerNodeStats = std::unordered_map<uint16_t, BucketSpacesStats>; private: - PerNodeStats _stats; + PerNodeStats _node_stats; + NodeMaintenanceStats _total_stats; static const NodeMaintenanceStats _emptyNodeMaintenanceStats; public: @@ -58,23 +59,28 @@ public: ~NodeMaintenanceStatsTracker(); void incMovingOut(uint16_t node, document::BucketSpace bucketSpace) { - ++_stats[node][bucketSpace].movingOut; + ++_node_stats[node][bucketSpace].movingOut; + ++_total_stats.movingOut; } void incSyncing(uint16_t node, document::BucketSpace bucketSpace) { - ++_stats[node][bucketSpace].syncing; + ++_node_stats[node][bucketSpace].syncing; + ++_total_stats.syncing; } void incCopyingIn(uint16_t node, document::BucketSpace bucketSpace) { - ++_stats[node][bucketSpace].copyingIn; + ++_node_stats[node][bucketSpace].copyingIn; + ++_total_stats.copyingIn; } void incCopyingOut(uint16_t node, document::BucketSpace bucketSpace) { - ++_stats[node][bucketSpace].copyingOut; + ++_node_stats[node][bucketSpace].copyingOut; + ++_total_stats.copyingOut; } void incTotal(uint16_t node, document::BucketSpace bucketSpace) { - ++_stats[node][bucketSpace].total; + ++_node_stats[node][bucketSpace].total; + ++_total_stats.total; } /** @@ -82,8 +88,8 @@ public: * if none have been recorded yet */ const NodeMaintenanceStats& forNode(uint16_t node, document::BucketSpace bucketSpace) const { - auto nodeItr = _stats.find(node); - if (nodeItr != _stats.end()) { + auto nodeItr = _node_stats.find(node); + if (nodeItr != _node_stats.end()) { auto bucketSpaceItr = nodeItr->second.find(bucketSpace); if (bucketSpaceItr != nodeItr->second.end()) { return bucketSpaceItr->second; @@ -93,11 +99,18 @@ public: } const PerNodeStats& perNodeStats() const { - return _stats; + return _node_stats; + } + + // Note: the total statistics are across all replicas across all buckets across all bucket spaces. + // That means it's possible for a single bucket to count more than once, up to once per replica. + // So this should not be treated as a bucket-level statistic. + const NodeMaintenanceStats& total_replica_stats() const noexcept { + return _total_stats; } bool operator==(const NodeMaintenanceStatsTracker& rhs) const { - return _stats == rhs._stats; + return _node_stats == rhs._node_stats; } void merge(const NodeMaintenanceStatsTracker& rhs); }; diff --git a/storage/src/vespa/storage/distributor/ownership_transfer_safe_time_point_calculator.cpp b/storage/src/vespa/storage/distributor/ownership_transfer_safe_time_point_calculator.cpp index a6954c973b6..437b894a190 100644 --- a/storage/src/vespa/storage/distributor/ownership_transfer_safe_time_point_calculator.cpp +++ b/storage/src/vespa/storage/distributor/ownership_transfer_safe_time_point_calculator.cpp @@ -7,8 +7,7 @@ namespace storage { namespace distributor { OwnershipTransferSafeTimePointCalculator::TimePoint -OwnershipTransferSafeTimePointCalculator::safeTimePoint( - TimePoint now) const +OwnershipTransferSafeTimePointCalculator::safeTimePoint(TimePoint now) const { if (_max_cluster_clock_skew.count() == 0) { return TimePoint{}; @@ -27,8 +26,7 @@ OwnershipTransferSafeTimePointCalculator::safeTimePoint( // adding the max skew. This prevents generating time stamps within // the same whole second as another distributor already has done for // any of the buckets a node now owns. - auto now_sec = std::chrono::duration_cast<std::chrono::seconds>( - now.time_since_epoch()); + auto now_sec = std::chrono::duration_cast<std::chrono::seconds>(now.time_since_epoch()); return TimePoint(now_sec + std::chrono::seconds(1) + _max_cluster_clock_skew); } diff --git a/storage/src/vespa/storage/distributor/ownership_transfer_safe_time_point_calculator.h b/storage/src/vespa/storage/distributor/ownership_transfer_safe_time_point_calculator.h index 407757d83ff..79ba36052ae 100644 --- a/storage/src/vespa/storage/distributor/ownership_transfer_safe_time_point_calculator.h +++ b/storage/src/vespa/storage/distributor/ownership_transfer_safe_time_point_calculator.h @@ -4,8 +4,7 @@ #include <chrono> -namespace storage { -namespace distributor { +namespace storage::distributor { /** * When bucket ownership changes in a cluster, there exists a time period @@ -25,7 +24,6 @@ namespace distributor { * equal to or lower than this. The stop-gap also breaks down if, in fact, * the clock skew is higher than the expected one. * - * This is an interface to avoid having to do real wall-clocks in unit tests. */ class OwnershipTransferSafeTimePointCalculator { std::chrono::seconds _max_cluster_clock_skew; @@ -47,4 +45,3 @@ public: }; } -} diff --git a/storage/src/vespa/storage/distributor/top_level_bucket_db_updater.cpp b/storage/src/vespa/storage/distributor/top_level_bucket_db_updater.cpp index ac97dde6a0c..18a53c9e8ba 100644 --- a/storage/src/vespa/storage/distributor/top_level_bucket_db_updater.cpp +++ b/storage/src/vespa/storage/distributor/top_level_bucket_db_updater.cpp @@ -50,19 +50,19 @@ TopLevelBucketDBUpdater::TopLevelBucketDBUpdater(const DistributorNodeContext& n _stale_reads_enabled(false) { // FIXME STRIPE top-level Distributor needs a proper way to track the current cluster state bundle! - propagate_active_state_bundle_internally(); + propagate_active_state_bundle_internally(true); // We're just starting up so assume ownership transfer. bootstrap_distribution_config(bootstrap_distribution); } TopLevelBucketDBUpdater::~TopLevelBucketDBUpdater() = default; void -TopLevelBucketDBUpdater::propagate_active_state_bundle_internally() { +TopLevelBucketDBUpdater::propagate_active_state_bundle_internally(bool has_bucket_ownership_transfer) { for (auto& elem : _op_ctx.bucket_space_states()) { elem.second->set_cluster_state(_active_state_bundle.getDerivedClusterState(elem.first)); } if (_state_activation_listener) { - _state_activation_listener->on_cluster_state_bundle_activated(_active_state_bundle); + _state_activation_listener->on_cluster_state_bundle_activated(_active_state_bundle, has_bucket_ownership_transfer); } } @@ -412,7 +412,7 @@ TopLevelBucketDBUpdater::enable_current_cluster_state_bundle_in_distributor_and_ _active_state_bundle = _pending_cluster_state->getNewClusterStateBundle(); guard.enable_cluster_state_bundle(state, _pending_cluster_state->hasBucketOwnershipTransfer()); - propagate_active_state_bundle_internally(); + propagate_active_state_bundle_internally(_pending_cluster_state->hasBucketOwnershipTransfer()); LOG(debug, "TopLevelBucketDBUpdater finished processing state %s", state.getBaselineClusterState()->toString().c_str()); @@ -425,7 +425,7 @@ void TopLevelBucketDBUpdater::simulate_cluster_state_bundle_activation(const lib guard->enable_cluster_state_bundle(activated_state, has_bucket_ownership_transfer); _active_state_bundle = activated_state; - propagate_active_state_bundle_internally(); + propagate_active_state_bundle_internally(has_bucket_ownership_transfer); } void diff --git a/storage/src/vespa/storage/distributor/top_level_bucket_db_updater.h b/storage/src/vespa/storage/distributor/top_level_bucket_db_updater.h index e01ea30cbda..3b5d5e4a453 100644 --- a/storage/src/vespa/storage/distributor/top_level_bucket_db_updater.h +++ b/storage/src/vespa/storage/distributor/top_level_bucket_db_updater.h @@ -75,7 +75,6 @@ public: private: friend class DistributorStripeTestUtil; - friend class DistributorTestUtil; friend class TopLevelDistributorTestUtil; // Only to be used by tests that want to ensure both the TopLevelBucketDBUpdater _and_ the Distributor // components agree on the currently active cluster state bundle. @@ -104,7 +103,7 @@ private: void enable_current_cluster_state_bundle_in_distributor_and_stripes(StripeAccessGuard& guard); void add_current_state_to_cluster_state_history(); - void propagate_active_state_bundle_internally(); + void propagate_active_state_bundle_internally(bool has_bucket_ownership_transfer); void maybe_inject_simulated_db_pruning_delay(); void maybe_inject_simulated_db_merging_delay(); diff --git a/storage/src/vespa/storage/distributor/top_level_distributor.cpp b/storage/src/vespa/storage/distributor/top_level_distributor.cpp index 5f8f05c3ee0..a8a87e9f03c 100644 --- a/storage/src/vespa/storage/distributor/top_level_distributor.cpp +++ b/storage/src/vespa/storage/distributor/top_level_distributor.cpp @@ -10,7 +10,6 @@ #include "distributor_stripe_pool.h" #include "distributor_stripe_thread.h" #include "distributor_total_metrics.h" -#include "idealstatemetricsset.h" #include "multi_threaded_stripe_access_guard.h" #include "operation_sequencer.h" #include "ownership_transfer_safe_time_point_calculator.h" @@ -74,6 +73,8 @@ TopLevelDistributor::TopLevelDistributor(DistributorComponentRegister& compReg, _stripe_scan_stats(), _last_host_info_send_time(), _host_info_send_delay(1000ms), + _maintenance_safe_time_point(), + _maintenance_safe_time_delay(1s), _tickResult(framework::ThreadWaitInfo::NO_MORE_CRITICAL_WORK_KNOWN), _metricUpdateHook(*this), _hostInfoReporter(*this, *this), @@ -421,6 +422,8 @@ TopLevelDistributor::doCriticalTick([[maybe_unused]] framework::ThreadIndex idx) fetch_external_messages(); // Propagates any new configs down to stripe(s) enable_next_config_if_changed(); + un_inhibit_maintenance_if_safe_time_passed(); + return _tickResult; } @@ -446,14 +449,32 @@ TopLevelDistributor::enable_next_config_if_changed() guard->update_total_distributor_config(_component.total_distributor_config_sp()); } _hostInfoReporter.enableReporting(config().getEnableHostInfoReporting()); + _maintenance_safe_time_delay = _total_config->getMaxClusterClockSkew(); _current_internal_config_generation = _component.internal_config_generation(); } } void +TopLevelDistributor::un_inhibit_maintenance_if_safe_time_passed() +{ + if (_maintenance_safe_time_point.time_since_epoch().count() != 0) { + using TimePoint = OwnershipTransferSafeTimePointCalculator::TimePoint; + const auto now = TimePoint(std::chrono::seconds(_component.clock().getTimeInSeconds().getTime())); + if (now >= _maintenance_safe_time_point) { + // Thread safe. Relaxed store is fine; stripes will eventually observe new flag status. + for (auto& stripe : _stripes) { + stripe->inhibit_non_activation_maintenance_operations(false); + } + _maintenance_safe_time_point = TimePoint{}; + LOG(debug, "Marked all stripes as no longer inhibiting non-activation maintenance operations"); + } + } +} + +void TopLevelDistributor::notify_stripe_wants_to_send_host_info(uint16_t stripe_index) { - // TODO STRIPE assert(_done_initializing); (can't currently do due to some unit test restrictions; uncomment and find out) + assert(_done_initializing); LOG(debug, "Stripe %u has signalled an intent to send host info out-of-band", stripe_index); std::lock_guard lock(_stripe_scan_notify_mutex); assert(stripe_index < _stripe_scan_stats.size()); @@ -500,13 +521,25 @@ TopLevelDistributor::send_host_info_if_appropriate() } void -TopLevelDistributor::on_cluster_state_bundle_activated(const lib::ClusterStateBundle& new_bundle) +TopLevelDistributor::on_cluster_state_bundle_activated(const lib::ClusterStateBundle& new_bundle, + bool has_bucket_ownership_transfer) { lib::Node my_node(lib::NodeType::DISTRIBUTOR, getDistributorIndex()); if (!_done_initializing && (new_bundle.getBaselineClusterState()->getNodeState(my_node).getState() == lib::State::UP)) { _done_initializing = true; _done_init_handler.notifyDoneInitializing(); } + if (has_bucket_ownership_transfer && _maintenance_safe_time_delay.count() > 0) { + OwnershipTransferSafeTimePointCalculator safe_time_calc(_maintenance_safe_time_delay); + using TimePoint = OwnershipTransferSafeTimePointCalculator::TimePoint; + const auto now = TimePoint(std::chrono::milliseconds(_component.getClock().getTimeInMillis().getTime())); + _maintenance_safe_time_point = safe_time_calc.safeTimePoint(now); + // All stripes are in a waiting pattern and will observe this on their next tick. + // Memory visibility enforced by all stripes being held under a mutex by our caller. + for (auto& stripe : _stripes) { + stripe->inhibit_non_activation_maintenance_operations(true); + } + } LOG(debug, "Activated new state version in distributor: %s", new_bundle.toString().c_str()); } diff --git a/storage/src/vespa/storage/distributor/top_level_distributor.h b/storage/src/vespa/storage/distributor/top_level_distributor.h index 420d0df08ed..04f88c411d8 100644 --- a/storage/src/vespa/storage/distributor/top_level_distributor.h +++ b/storage/src/vespa/storage/distributor/top_level_distributor.h @@ -159,6 +159,7 @@ private: [[nodiscard]] bool work_was_done() const noexcept; void enableNextDistribution(); void propagateDefaultDistribution(std::shared_ptr<const lib::Distribution>); + void un_inhibit_maintenance_if_safe_time_passed(); void dispatch_to_main_distributor_thread_queue(const std::shared_ptr<api::StorageMessage>& msg); void fetch_external_messages(); @@ -171,7 +172,8 @@ private: uint32_t stripe_of_bucket_id(const document::BucketId& bucket_id, const api::StorageMessage& msg); // ClusterStateBundleActivationListener impl: - void on_cluster_state_bundle_activated(const lib::ClusterStateBundle&) override; + void on_cluster_state_bundle_activated(const lib::ClusterStateBundle& new_bundle, + bool has_bucket_ownership_transfer) override; struct StripeScanStats { bool wants_to_send_host_info = false; @@ -208,6 +210,10 @@ private: std::vector<StripeScanStats> _stripe_scan_stats; // Indices are 1-1 with _stripes entries std::chrono::steady_clock::time_point _last_host_info_send_time; std::chrono::milliseconds _host_info_send_delay; + // Ideally this would use steady_clock, but for now let's use the same semantics as + // feed blocking during safe time periods. + std::chrono::system_clock::time_point _maintenance_safe_time_point; + std::chrono::seconds _maintenance_safe_time_delay; framework::ThreadWaitInfo _tickResult; MetricUpdateHook _metricUpdateHook; DistributorHostInfoReporter _hostInfoReporter; diff --git a/storage/src/vespa/storage/storageserver/distributornode.cpp b/storage/src/vespa/storage/storageserver/distributornode.cpp index 620bd3571ce..d8558d4c620 100644 --- a/storage/src/vespa/storage/storageserver/distributornode.cpp +++ b/storage/src/vespa/storage/storageserver/distributornode.cpp @@ -131,7 +131,7 @@ DistributorNode::generate_unique_timestamp() _intra_second_pseudo_usec_counter); std::_Exit(65); } - assert(_intra_second_pseudo_usec_counter < 1'000'000); + assert(_intra_second_pseudo_usec_counter < 999'999); ++_intra_second_pseudo_usec_counter; } else { _timestamp_second_counter = now_seconds; diff --git a/storage/src/vespa/storage/storageserver/rpc/rpc_envelope_proto.h b/storage/src/vespa/storage/storageserver/rpc/rpc_envelope_proto.h index dbc048cd504..7e065ab0739 100644 --- a/storage/src/vespa/storage/storageserver/rpc/rpc_envelope_proto.h +++ b/storage/src/vespa/storage/storageserver/rpc/rpc_envelope_proto.h @@ -7,6 +7,6 @@ #pragma GCC diagnostic ignored "-Wsuggest-override" #endif -#include "rpc_envelope.pb.h" +#include <vespa/storage/storageserver/rpc/rpc_envelope.pb.h> #pragma GCC diagnostic pop diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h b/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h index 289e5dc355c..f5111ae8061 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h +++ b/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h @@ -8,9 +8,9 @@ #pragma GCC diagnostic ignored "-Wsuggest-override" #endif -#include "feed.pb.h" -#include "inspect.pb.h" -#include "visiting.pb.h" -#include "maintenance.pb.h" +#include <vespa/storageapi/mbusprot/feed.pb.h> +#include <vespa/storageapi/mbusprot/inspect.pb.h> +#include <vespa/storageapi/mbusprot/visiting.pb.h> +#include <vespa/storageapi/mbusprot/maintenance.pb.h> #pragma GCC diagnostic pop diff --git a/vdslib/src/main/java/com/yahoo/vdslib/state/ClusterState.java b/vdslib/src/main/java/com/yahoo/vdslib/state/ClusterState.java index c536fc0f4cd..8704b11fdfb 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/state/ClusterState.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/state/ClusterState.java @@ -3,12 +3,13 @@ package com.yahoo.vdslib.state; import com.yahoo.text.StringUtilities; import java.text.ParseException; -import java.util.ArrayList; +import java.util.BitSet; +import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.StringTokenizer; -import java.util.TreeMap; -import java.util.TreeSet; /** * Be careful about changing this class, as it mirrors the ClusterState in C++. @@ -18,25 +19,183 @@ public class ClusterState implements Cloneable { private static final NodeState DEFAULT_STORAGE_UP_NODE_STATE = new NodeState(NodeType.STORAGE, State.UP); private static final NodeState DEFAULT_DISTRIBUTOR_UP_NODE_STATE = new NodeState(NodeType.DISTRIBUTOR, State.UP); + private static final NodeState DEFAULT_STORAGE_DOWN_NODE_STATE = new NodeState(NodeType.STORAGE, State.DOWN); + private static final NodeState DEFAULT_DISTRIBUTOR_DOWN_NODE_STATE = new NodeState(NodeType.DISTRIBUTOR, State.DOWN); - private int version = 0; - private State state = State.DOWN; - // nodeStates maps each of the non-up nodes that have an index <= the node count for its type. - private Map<Node, NodeState> nodeStates = new TreeMap<>(); + /** + * Maintains a bitset where all non-down nodes have a bit set. All nodes that differs from defaultUp + * and defaultDown are store explicit in a hash map. + */ + private static class Nodes { + private int logicalNodeCount; + private final NodeType type; + private final BitSet upNodes; + private final Map<Integer, NodeState> nodeStates = new HashMap<>(); + Nodes(NodeType type) { + this.type = type; + upNodes = new BitSet(); + } + Nodes(Nodes b) { + logicalNodeCount = b.logicalNodeCount; + type = b.type; + upNodes = (BitSet) b.upNodes.clone(); + b.nodeStates.forEach((key, value) -> nodeStates.put(key, value.clone())); + } + + void updateMaxIndex(int index) { + if (index > logicalNodeCount) { + upNodes.set(logicalNodeCount, index); + logicalNodeCount = index; + } + } + + int getLogicalNodeCount() { return logicalNodeCount; } + + NodeState getNodeState(int index) { + NodeState ns = nodeStates.get(index); + if (ns != null) return ns; + return (index >= getLogicalNodeCount() || ! upNodes.get(index)) + ? new NodeState(type, State.DOWN) + : new NodeState(type, State.UP); + } + + private void validateInput(Node node, NodeState ns) { + ns.verifyValidInSystemState(node.getType()); + if (node.getType() != type) { + throw new IllegalArgumentException("NodeType '" + node.getType() + "' differs from '" + type + "'"); + } + } + + void setNodeState(Node node, NodeState ns) { + validateInput(node, ns); + int index = node.getIndex(); + if (index >= logicalNodeCount) { + logicalNodeCount = index + 1; + } + setNodeStateInternal(index, ns); + } + + void addNodeState(Node node, NodeState ns) { + validateInput(node, ns); + int index = node.getIndex(); + updateMaxIndex(index + 1); + setNodeStateInternal(index, ns); + } + + private static boolean equalsWithDescription(NodeState a, NodeState b) { + // This is due to NodeState.equals considers semantic equality, and description is not part of that. + return a.equals(b) && ((a.getState() != State.DOWN) || a.getDescription().equals(b.getDescription())); + } + + private void setNodeStateInternal(int index, NodeState ns) { + nodeStates.remove(index); + if (ns.getState() == State.DOWN) { + upNodes.clear(index); + if ( ! equalsWithDescription(defaultDown(), ns)) { + nodeStates.put(index, ns); + } + } else { + upNodes.set(index); + if ( ! equalsWithDescription(defaultUp(), ns)) { + nodeStates.put(index, ns); + } + } + } + + boolean similarToImpl(Nodes other, final NodeStateCmp nodeStateCmp) { + // TODO verify behavior of C++ impl against this + if (logicalNodeCount != other.logicalNodeCount) return false; + if (type != other.type) return false; + if ( ! upNodes.equals(other.upNodes)) return false; + for (Integer node : unionNodeSetWith(other.nodeStates.keySet())) { + final NodeState lhs = nodeStates.get(node); + final NodeState rhs = other.nodeStates.get(node); + if (!nodeStateCmp.similar(type, lhs, rhs)) { + return false; + } + } + return true; + } + + private Set<Integer> unionNodeSetWith(final Set<Integer> otherNodes) { + final Set<Integer> unionNodeSet = new HashSet<>(nodeStates.keySet()); + unionNodeSet.addAll(otherNodes); + return unionNodeSet; + } + + @Override + public String toString() { return toString(false); } + + String toString(boolean verbose) { + StringBuilder sb = new StringBuilder(); + + int nodeCount = verbose ? getLogicalNodeCount() : upNodes.length(); + if ( nodeCount > 0 ) { + sb.append(type == NodeType.DISTRIBUTOR ? " distributor:" : " storage:").append(nodeCount); + for (int i = 0; i < nodeCount; i++) { + String nodeState = getNodeState(i).serialize(i, verbose); + if (!nodeState.isEmpty()) { + sb.append(' ').append(nodeState); + } + } + } + return sb.toString(); + } + + @Override + public boolean equals(Object obj) { + if (! (obj instanceof Nodes)) return false; + Nodes b = (Nodes) obj; + if (logicalNodeCount != b.logicalNodeCount) return false; + if (type != b.type) return false; + if (!upNodes.equals(b.upNodes)) return false; + if (!nodeStates.equals(b.nodeStates)) return false; + return true; + } - // TODO: Change to one count for distributor and one for storage, rather than an array - // TODO: RenameFunction, this is not the highest node count but the highest index - private ArrayList<Integer> nodeCount = new ArrayList<>(2); + @Override + public int hashCode() { + return Objects.hash(logicalNodeCount, type, nodeStates, upNodes); + } + private NodeState defaultDown() { + return type == NodeType.STORAGE + ? DEFAULT_STORAGE_DOWN_NODE_STATE + : DEFAULT_DISTRIBUTOR_DOWN_NODE_STATE; + } + private NodeState defaultUp() { + return defaultUpNodeState(type); + } + } + private int version = 0; + private State state = State.DOWN; private String description = ""; private int distributionBits = 16; + private final Nodes distributorNodes; + private final Nodes storageNodes; + public ClusterState(String serialized) throws ParseException { - nodeCount.add(0); - nodeCount.add(0); + distributorNodes = new Nodes(NodeType.DISTRIBUTOR); + storageNodes = new Nodes(NodeType.STORAGE); deserialize(serialized); } + public ClusterState(ClusterState b) { + version = b.version; + state = b.state; + description = b.description; + distributionBits = b.distributionBits; + distributorNodes = new Nodes(b.distributorNodes); + storageNodes = new Nodes(b.storageNodes); + } + + private Nodes getNodes(NodeType type) { + return (type == NodeType.STORAGE) + ? storageNodes + : (type == NodeType.DISTRIBUTOR) ? distributorNodes : null; + } + /** * Parse a given cluster state string into a returned ClusterState instance, wrapping any * parse exceptions in a RuntimeException. @@ -54,20 +213,7 @@ public class ClusterState implements Cloneable { } public ClusterState clone() { - try{ - ClusterState state = (ClusterState) super.clone(); - state.nodeStates = new TreeMap<>(); - for (Map.Entry<Node, NodeState> entry : nodeStates.entrySet()) { - state.nodeStates.put(entry.getKey(), entry.getValue().clone()); - } - state.nodeCount = new ArrayList<>(2); - state.nodeCount.add(nodeCount.get(0)); - state.nodeCount.add(nodeCount.get(1)); - return state; - } catch (CloneNotSupportedException e) { - assert(false); // Should never happen - return null; - } + return new ClusterState(this); } @Override @@ -77,8 +223,8 @@ public class ClusterState implements Cloneable { if (version != other.version || !state.equals(other.state) || distributionBits != other.distributionBits - || !nodeCount.equals(other.nodeCount) - || !nodeStates.equals(other.nodeStates)) + || !distributorNodes.equals(other.distributorNodes) + || !storageNodes.equals(other.storageNodes)) { return false; } @@ -87,7 +233,7 @@ public class ClusterState implements Cloneable { @Override public int hashCode() { - return java.util.Objects.hash(version, state, distributionBits, nodeCount, nodeStates); + return java.util.Objects.hash(version, state, distributionBits, distributorNodes, storageNodes); } @FunctionalInterface @@ -106,7 +252,7 @@ public class ClusterState implements Cloneable { return similarToImpl(other, this::normalizedNodeStateSimilarToIgnoringInitProgress); } - private boolean similarToImpl(final ClusterState other, final NodeStateCmp nodeStateCmp) { + private boolean similarToImpl(ClusterState other, final NodeStateCmp nodeStateCmp) { if (other == this) { return true; // We're definitely similar to ourselves. } @@ -119,23 +265,11 @@ public class ClusterState implements Cloneable { if (!metaInformationSimilarTo(other)) { return false; } - // TODO verify behavior of C++ impl against this - for (Node node : unionNodeSetWith(other.nodeStates.keySet())) { - final NodeState lhs = nodeStates.get(node); - final NodeState rhs = other.nodeStates.get(node); - if (!nodeStateCmp.similar(node.getType(), lhs, rhs)) { - return false; - } - } + if ( !distributorNodes.similarToImpl(other.distributorNodes, nodeStateCmp)) return false; + if ( !storageNodes.similarToImpl(other.storageNodes, nodeStateCmp)) return false; return true; } - private Set<Node> unionNodeSetWith(final Set<Node> otherNodes) { - final Set<Node> unionNodeSet = new TreeSet<>(nodeStates.keySet()); - unionNodeSet.addAll(otherNodes); - return unionNodeSet; - } - private boolean metaInformationSimilarTo(final ClusterState other) { if (version != other.version || !state.equals(other.state)) { return false; @@ -143,7 +277,7 @@ public class ClusterState implements Cloneable { if (distributionBits != other.distributionBits) { return false; } - return nodeCount.equals(other.nodeCount); + return true; } private boolean normalizedNodeStateSimilarTo(final NodeType nodeType, final NodeState lhs, final NodeState rhs) { @@ -178,12 +312,7 @@ public class ClusterState implements Cloneable { void addNodeState() throws ParseException { if (!empty) { NodeState ns = NodeState.deserialize(node.getType(), sb.toString()); - if (!ns.equals(defaultUpNodeState(node.getType()))) { - nodeStates.put(node, ns); - } - if (nodeCount.get(node.getType().ordinal()) <= node.getIndex()) { - nodeCount.set(node.getType().ordinal(), node.getIndex() + 1); - } + getNodes(node.getType()).addNodeState(node, ns); } empty = true; sb = new StringBuilder(); @@ -257,9 +386,7 @@ public class ClusterState implements Cloneable { } catch (Exception e) { throw new ParseException("Illegal node count '" + value + "' in state: " + serialized, 0); } - if (nodeCount > this.nodeCount.get(nodeType.ordinal())) { - this.nodeCount.set(nodeType.ordinal(), nodeCount); - } + getNodes(nodeType).updateMaxIndex(nodeCount); continue; } int dot2 = key.indexOf('.', dot + 1); @@ -269,8 +396,8 @@ public class ClusterState implements Cloneable { } else { node = new Node(nodeType, Integer.valueOf(key.substring(dot + 1, dot2))); } - if (node.getIndex() >= this.nodeCount.get(nodeType.ordinal())) { - throw new ParseException("Cannot index " + nodeType + " node " + node.getIndex() + " of " + this.nodeCount.get(nodeType.ordinal()) + " in state: " + serialized, 0); + if (node.getIndex() >= getNodeCount(nodeType)) { + throw new ParseException("Cannot index " + nodeType + " node " + node.getIndex() + " of " + getNodeCount(nodeType) + " in state: " + serialized, 0); } if (!nodeData.node.equals(node)) { nodeData.addNodeState(); @@ -289,7 +416,6 @@ public class ClusterState implements Cloneable { // Ignore unknown nodeStates } nodeData.addNodeState(); - removeLastNodesDownWithoutReason(); } public String getTextualDifference(ClusterState other) { @@ -363,7 +489,7 @@ public class ClusterState implements Cloneable { * E.g. if node X is down and without description, but nodex X-1 is up, then Y is 1. * The node count for distributors is then X + 1 - Y. */ - public int getNodeCount(NodeType type) { return nodeCount.get(type.ordinal()); } + public int getNodeCount(NodeType type) { return getNodes(type).getLogicalNodeCount(); } /** * Returns the state of a node. @@ -371,9 +497,7 @@ public class ClusterState implements Cloneable { * and DOWN otherwise. */ public NodeState getNodeState(Node node) { - if (node.getIndex() >= nodeCount.get(node.getType().ordinal())) - return new NodeState(node.getType(), State.DOWN); - return nodeStates.getOrDefault(node, new NodeState(node.getType(), State.UP)); + return getNodes(node.getType()).getNodeState(node.getIndex()); } /** @@ -383,35 +507,7 @@ public class ClusterState implements Cloneable { */ public void setNodeState(Node node, NodeState newState) { newState.verifyValidInSystemState(node.getType()); - if (node.getIndex() >= nodeCount.get(node.getType().ordinal())) { - for (int i= nodeCount.get(node.getType().ordinal()); i<node.getIndex(); ++i) { - nodeStates.put(new Node(node.getType(), i), new NodeState(node.getType(), State.DOWN)); - } - nodeCount.set(node.getType().ordinal(), node.getIndex() + 1); - } - if (newState.equals(new NodeState(node.getType(), State.UP))) { - nodeStates.remove(node); - } else { - nodeStates.put(node, newState); - } - if (newState.getState().equals(State.DOWN)) { - // We might be setting the last node down, so we can remove some states - removeLastNodesDownWithoutReason(); - } - } - - private void removeLastNodesDownWithoutReason() { - for (NodeType nodeType : NodeType.values()) { - for (int index = nodeCount.get(nodeType.ordinal()) - 1; index >= 0; --index) { - Node node = new Node(nodeType, index); - NodeState nodeState = nodeStates.get(node); - if (nodeState == null) break; // Node not existing is up - if ( ! nodeState.getState().equals(State.DOWN)) break; // Node not down can not be removed - if (nodeState.hasDescription()) break; // Node have reason to be down. Don't remove node as we will forget reason - nodeStates.remove(node); - nodeCount.set(nodeType.ordinal(), node.getIndex()); - } - } + getNodes(node.getType()).setNodeState(node, newState); } public String getDescription() { return description; } @@ -440,35 +536,9 @@ public class ClusterState implements Cloneable { sb.append(" bits:").append(distributionBits); } - int distributorNodeCount = getNodeCount(NodeType.DISTRIBUTOR); - int storageNodeCount = getNodeCount(NodeType.STORAGE); - // If not printing verbose, we're not printing descriptions, so we can remove tailing nodes that are down that has descriptions too - if (!verbose) { - while (distributorNodeCount > 0 && getNodeState(new Node(NodeType.DISTRIBUTOR, distributorNodeCount - 1)).getState().equals(State.DOWN)) --distributorNodeCount; - while (storageNodeCount > 0 && getNodeState(new Node(NodeType.STORAGE, storageNodeCount - 1)).getState().equals(State.DOWN)) --storageNodeCount; - } - if (distributorNodeCount > 0){ - sb.append(" distributor:").append(distributorNodeCount); - for (Map.Entry<Node, NodeState> entry : nodeStates.entrySet()) { - if (entry.getKey().getType().equals(NodeType.DISTRIBUTOR) && entry.getKey().getIndex() < distributorNodeCount) { - String nodeState = entry.getValue().serialize(entry.getKey().getIndex(), verbose); - if (!nodeState.isEmpty()) { - sb.append(' ').append(nodeState); - } - } - } - } - if (storageNodeCount > 0){ - sb.append(" storage:").append(storageNodeCount); - for (Map.Entry<Node, NodeState> entry : nodeStates.entrySet()) { - if (entry.getKey().getType().equals(NodeType.STORAGE) && entry.getKey().getIndex() < storageNodeCount) { - String nodeState = entry.getValue().serialize(entry.getKey().getIndex(), verbose); - if (!nodeState.isEmpty()) { - sb.append(' ').append(nodeState); - } - } - } - } + sb.append(distributorNodes.toString(verbose)); + sb.append(storageNodes.toString(verbose)); + if (sb.length() > 0) { // Remove first space if not empty sb.deleteCharAt(0); } diff --git a/vdslib/src/main/java/com/yahoo/vdslib/state/NodeState.java b/vdslib/src/main/java/com/yahoo/vdslib/state/NodeState.java index 0edf30bc1ff..80cf87b7597 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/state/NodeState.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/state/NodeState.java @@ -51,6 +51,7 @@ public class NodeState implements Cloneable { public boolean equals(Object o) { if (!(o instanceof NodeState)) { return false; } NodeState ns = (NodeState) o; + // Note that 'description' is not considered as it carries semantics. if (state != ns.state || Math.abs(capacity - ns.capacity) > 0.0000000001 || Math.abs(initProgress - ns.initProgress) > 0.0000000001 diff --git a/vdslib/src/test/java/com/yahoo/vdslib/state/ClusterStateTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/state/ClusterStateTestCase.java index dbc8888cfda..956ea216a44 100644 --- a/vdslib/src/test/java/com/yahoo/vdslib/state/ClusterStateTestCase.java +++ b/vdslib/src/test/java/com/yahoo/vdslib/state/ClusterStateTestCase.java @@ -4,6 +4,7 @@ package com.yahoo.vdslib.state; import org.junit.Test; import java.text.ParseException; +import java.util.BitSet; import java.util.function.BiFunction; import static org.junit.Assert.assertEquals; @@ -311,6 +312,17 @@ public class ClusterStateTestCase{ } @Test + public void testBitSet() { + BitSet b = new BitSet(); + assertEquals(0, b.length()); + b.set(7); + b.set(107); + assertEquals(108, b.length()); + b.clear(107); + assertEquals(8, b.length()); + } + + @Test public void testVersionAndClusterStates() throws ParseException { ClusterState state = new ClusterState("version:4 cluster:i distributor:2 .1.s:i storage:2 .0.s:i .0.i:0.345"); assertEquals(4, state.getVersion()); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java index acf580d7e1b..3acd1c34d51 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java @@ -34,7 +34,7 @@ import java.util.logging.Logger; */ public abstract class ClientBase implements AutoCloseable { - private static final Logger logger = Logger.getLogger(ClientBase.class.getName()); + protected final Logger logger = Logger.getLogger(getClass().getName()); private static final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule()); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java index 297852e9584..769a2a54c95 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.client.zms; -import com.yahoo.io.IOUtils; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzGroup; import com.yahoo.vespa.athenz.api.AthenzIdentity; @@ -18,7 +17,7 @@ import com.yahoo.vespa.athenz.client.zms.bindings.AssertionEntity; import com.yahoo.vespa.athenz.client.zms.bindings.DomainListResponseEntity; import com.yahoo.vespa.athenz.client.zms.bindings.MembershipEntity; import com.yahoo.vespa.athenz.client.zms.bindings.PolicyEntity; -import com.yahoo.vespa.athenz.client.zms.bindings.ProviderResourceGroupRolesRequestEntity; +import com.yahoo.vespa.athenz.client.zms.bindings.ResourceGroupRolesEntity; import com.yahoo.vespa.athenz.client.zms.bindings.ResponseListEntity; import com.yahoo.vespa.athenz.client.zms.bindings.RoleEntity; import com.yahoo.vespa.athenz.client.zms.bindings.ServiceEntity; @@ -33,11 +32,8 @@ import org.apache.http.entity.StringEntity; import org.apache.http.message.BasicHeader; import javax.net.ssl.SSLContext; -import java.io.IOException; import java.net.URI; -import java.nio.charset.StandardCharsets; import java.time.Instant; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -104,7 +100,7 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { HttpUriRequest request = RequestBuilder.put() .setUri(uri) .addHeader(createCookieHeaderWithOktaTokens(identityToken, accessToken)) - .setEntity(toJsonStringEntity(new ProviderResourceGroupRolesRequestEntity(providerService, tenantDomain, roleActions, resourceGroup))) + .setEntity(toJsonStringEntity(new ResourceGroupRolesEntity(providerService, tenantDomain, roleActions, resourceGroup))) .build(); execute(request, response -> readEntity(response, Void.class)); // Note: The ZMS API will actually return a json object that is similar to ProviderResourceGroupRolesRequestEntity } @@ -121,6 +117,31 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { } @Override + public void createTenantResourceGroup(AthenzDomain tenantDomain, AthenzIdentity provider, String resourceGroup, + Set<RoleAction> roleActions) { + URI uri = zmsUrl.resolve(String.format("domain/%s/service/%s/tenant/%s/resourceGroup/%s", + provider.getDomainName(), provider.getName(), tenantDomain.getName(), resourceGroup)); + HttpUriRequest request = RequestBuilder.put() + .setUri(uri) + .setEntity(toJsonStringEntity( + new ResourceGroupRolesEntity(provider, tenantDomain, roleActions, resourceGroup))) + .build(); + execute(request, response -> readEntity(response, Void.class)); + } + + @Override + public Set<RoleAction> getTenantResourceGroups(AthenzDomain tenantDomain, AthenzIdentity provider, + String resourceGroup) { + URI uri = zmsUrl.resolve(String.format("domain/%s/service/%s/tenant/%s/resourceGroup/%s", + provider.getDomainName(), provider.getName(), tenantDomain.getName(), resourceGroup)); + HttpUriRequest request = RequestBuilder.get() + .setUri(uri) + .build(); + ResourceGroupRolesEntity result = execute(request, response -> readEntity(response, ResourceGroupRolesEntity.class)); + return result.roles.stream().map(rgr -> new RoleAction(rgr.role, rgr.action)).collect(Collectors.toSet()); + } + + @Override public void addRoleMember(AthenzRole role, AthenzIdentity member, Optional<String> reason) { URI uri = zmsUrl.resolve(String.format("domain/%s/role/%s/member/%s", role.domain().getName(), role.roleName(), member.getFullName())); MembershipEntity membership = new MembershipEntity.RoleMembershipEntity(member.getFullName(), true, role.roleName(), null); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java index 7dd0585bfd4..b1c26923113 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java @@ -34,6 +34,12 @@ public interface ZmsClient extends AutoCloseable { void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzIdentity providerService, String resourceGroup, OktaIdentityToken identityToken, OktaAccessToken accessToken); + /** For manual tenancy provisioning - only creates roles/policies on provider domain */ + void createTenantResourceGroup(AthenzDomain tenantDomain, AthenzIdentity provider, String resourceGroup, + Set<RoleAction> roleActions); + + Set<RoleAction> getTenantResourceGroups(AthenzDomain tenantDomain, AthenzIdentity provider, String resourceGroup); + void addRoleMember(AthenzRole role, AthenzIdentity member, Optional<String> reason); void deleteRoleMember(AthenzRole role, AthenzIdentity member); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ProviderResourceGroupRolesRequestEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ResourceGroupRolesEntity.java index a67bd4dcad6..865dc8c02cb 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ProviderResourceGroupRolesRequestEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ResourceGroupRolesEntity.java @@ -1,39 +1,53 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.client.zms.bindings; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zms.RoleAction; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; import static java.util.stream.Collectors.toList; /** * @author bjorncs */ -public class ProviderResourceGroupRolesRequestEntity { +@JsonIgnoreProperties(ignoreUnknown = true) +public class ResourceGroupRolesEntity { @JsonProperty("domain") - private final String domain; + public final String domain; @JsonProperty("service") - private final String service; + public final String service; @JsonProperty("tenant") - private final String tenant; + public final String tenant; @JsonProperty("roles") - private final List<TenantRoleAction> roles; + public final List<TenantRoleAction> roles; @JsonProperty("resourceGroup") - private final String resourceGroup; + public final String resourceGroup; - public ProviderResourceGroupRolesRequestEntity(AthenzIdentity providerService, AthenzDomain tenantDomain, Set<RoleAction> rolesActions, String resourceGroup) { + @JsonCreator + public ResourceGroupRolesEntity(@JsonProperty("domain") String domain, + @JsonProperty("service") String service, + @JsonProperty("tenant") String tenant, + @JsonProperty("roles") List<TenantRoleAction> roles, + @JsonProperty("resourceGroup") String resourceGroup) { + this.domain = domain; + this.service = service; + this.tenant = tenant; + this.roles = roles; + this.resourceGroup = resourceGroup; + } + + public ResourceGroupRolesEntity(AthenzIdentity providerService, AthenzDomain tenantDomain, Set<RoleAction> rolesActions, String resourceGroup) { this.domain = providerService.getDomainName(); this.service = providerService.getName(); this.tenant = tenantDomain.getName(); @@ -43,10 +57,10 @@ public class ProviderResourceGroupRolesRequestEntity { public static class TenantRoleAction { @JsonProperty("role") - private final String role; + public final String role; @JsonProperty("action") - private final String action; + public final String action; public TenantRoleAction(String role, String action) { this.role = role; diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java index 0089499701f..6fea2d3faa4 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java @@ -22,18 +22,19 @@ public interface FeedClient extends Closeable { /** * Send a document put with the given parameters, returning a future with the result of the operation. * Exceptional completion will use be an instance of {@link FeedException} or one of its sub-classes. - * */ + */ CompletableFuture<Result> put(DocumentId documentId, String documentJson, OperationParameters params); /** * Send a document update with the given parameters, returning a future with the result of the operation. * Exceptional completion will use be an instance of {@link FeedException} or one of its sub-classes. - * */ + */ CompletableFuture<Result> update(DocumentId documentId, String updateJson, OperationParameters params); - /** Send a document remove with the given parameters, returning a future with the result of the operation. - * Exceptional completion will use be an instance of {@link FeedException} or one of its sub-classes. - * */ + /** + * Send a document remove with the given parameters, returning a future with the result of the operation. + * Exceptional completion will use be an instance of {@link FeedException} or one of its sub-classes. + */ CompletableFuture<Result> remove(DocumentId documentId, OperationParameters params); /** Returns a snapshot of the stats for this feed client, such as requests made, and responses by status. */ diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java index 58112f86090..345177dcf36 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java @@ -163,9 +163,9 @@ class HttpRequestStrategy implements RequestStrategy { return true; } - breaker.failure(response); logResponse(FINE, response, request, attempt); if (response.code() == 500 || response.code() == 502 || response.code() == 504) { // Hopefully temporary errors. + breaker.failure(response); return retry(request, attempt); } diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java index 50bdde28180..89597a2352a 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java @@ -108,7 +108,10 @@ public abstract class AbstractVespaMojo extends AbstractMojo { private Optional<Path> apiKeyPath(String tenant) { if (!isNullOrBlank(apiKeyFile)) return Optional.of(Paths.get(apiKeyFile)); - Path cliApiKeyFile = Paths.get(System.getProperty("user.home"), ".vespa", tenant + ".api-key.pem"); + Path cliApiKeyFile = Optional.ofNullable(System.getenv("VESPA_CLI_HOME")) + .map(Paths::get) + .orElseGet(() -> Paths.get(System.getProperty("user.home"), ".vespa")) + .resolve(tenant + ".api-key.pem"); if (Files.exists(cliApiKeyFile)) return Optional.of(cliApiKeyFile); return Optional.empty(); diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index e5d5b8ba5b6..e68a37b15b6 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -1203,6 +1203,7 @@ "public com.yahoo.tensor.Tensor notEqual(com.yahoo.tensor.Tensor)", "public com.yahoo.tensor.Tensor approxEqual(com.yahoo.tensor.Tensor)", "public com.yahoo.tensor.Tensor bit(com.yahoo.tensor.Tensor)", + "public com.yahoo.tensor.Tensor hamming(com.yahoo.tensor.Tensor)", "public com.yahoo.tensor.Tensor avg()", "public com.yahoo.tensor.Tensor avg(java.lang.String)", "public com.yahoo.tensor.Tensor avg(java.util.List)", @@ -2189,6 +2190,22 @@ ], "fields": [] }, + "com.yahoo.tensor.functions.ScalarFunctions$Hamming": { + "superClass": "java.lang.Object", + "interfaces": [ + "java.util.function.DoubleBinaryOperator" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public static double hamming(double, double)", + "public double applyAsDouble(double, double)", + "public java.lang.String toString()" + ], + "fields": [] + }, "com.yahoo.tensor.functions.ScalarFunctions$LeakyRelu": { "superClass": "java.lang.Object", "interfaces": [ @@ -2557,6 +2574,7 @@ "public static java.util.function.DoubleBinaryOperator pow()", "public static java.util.function.DoubleBinaryOperator squareddifference()", "public static java.util.function.DoubleBinaryOperator subtract()", + "public static java.util.function.DoubleBinaryOperator hamming()", "public static java.util.function.DoubleUnaryOperator abs()", "public static java.util.function.DoubleUnaryOperator acos()", "public static java.util.function.DoubleUnaryOperator asin()", diff --git a/vespajlib/src/main/java/com/yahoo/collections/CopyOnWriteHashMap.java b/vespajlib/src/main/java/com/yahoo/collections/CopyOnWriteHashMap.java index 857b7cc6acd..98370a8735a 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/CopyOnWriteHashMap.java +++ b/vespajlib/src/main/java/com/yahoo/collections/CopyOnWriteHashMap.java @@ -9,7 +9,7 @@ import java.util.Map; import java.util.Set; /** - * A hashmap wrapper which defers cloning of the enclosed map until it is written. + * A hashmap wrapper which defers cloning of the enclosed map until it is written to. * Use this to make clones cheap in maps which are often not further modified. * <p> * As with regular maps, this can only be used safely if the content of the map is immutable. diff --git a/vespajlib/src/main/java/com/yahoo/lang/MutableBoolean.java b/vespajlib/src/main/java/com/yahoo/lang/MutableBoolean.java index 17501b17bd0..877620547ba 100644 --- a/vespajlib/src/main/java/com/yahoo/lang/MutableBoolean.java +++ b/vespajlib/src/main/java/com/yahoo/lang/MutableBoolean.java @@ -22,6 +22,12 @@ public class MutableBoolean { public void orSet(boolean value) { this.value |= value; } + public boolean getAndSet(boolean newValue) { + boolean prev = value; + value = newValue; + return prev; + } + @Override public String toString() { return Boolean.toString(value); } diff --git a/vespajlib/src/main/java/com/yahoo/protect/Process.java b/vespajlib/src/main/java/com/yahoo/protect/Process.java index f3674f665b2..8038382c348 100644 --- a/vespajlib/src/main/java/com/yahoo/protect/Process.java +++ b/vespajlib/src/main/java/com/yahoo/protect/Process.java @@ -74,9 +74,13 @@ public final class Process { } } - public static void dumpHeap(String filePath, boolean live) throws IOException { + public static void dumpHeap(String filePath, boolean live) { log.log(Level.INFO, "Will dump the heap to '" + filePath + "', with the live = " + live); - getHotspotMXBean().dumpHeap(filePath, live); + try { + getHotspotMXBean().dumpHeap(filePath, live); + } catch (IOException e) { + log.log(Level.WARNING, "Failed writing heap dump:", e); + } } private static HotSpotDiagnosticMXBean getHotspotMXBean() throws IOException { diff --git a/vespajlib/src/main/java/com/yahoo/slime/ArrayValue.java b/vespajlib/src/main/java/com/yahoo/slime/ArrayValue.java index c8a7f2253bb..ac1a33eea0d 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/ArrayValue.java +++ b/vespajlib/src/main/java/com/yahoo/slime/ArrayValue.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.slime; - final class ArrayValue extends Value { private int capacity = 16; @@ -10,16 +9,16 @@ final class ArrayValue extends Value { private final SymbolTable names; public ArrayValue(SymbolTable names) { this.names = names; } - public final Type type() { return Type.ARRAY; } - public final int children() { return used; } - public final int entries() { return used; } - public final Value entry(int index) { + public Type type() { return Type.ARRAY; } + public int children() { return used; } + public int entries() { return used; } + public Value entry(int index) { return (index < used) ? values[index] : NixValue.invalid(); } - public final void accept(Visitor v) { v.visitArray(this); } + public void accept(Visitor v) { v.visitArray(this); } - public final void traverse(ArrayTraverser at) { + public void traverse(ArrayTraverser at) { for (int i = 0; i < used; i++) { at.entry(i, values[i]); } @@ -32,7 +31,7 @@ final class ArrayValue extends Value { System.arraycopy(v, 0, values, 0, used); } - protected final Value addLeaf(Value value) { + protected Value addLeaf(Value value) { if (used == capacity) { grow(); } @@ -40,6 +39,7 @@ final class ArrayValue extends Value { return value; } - public final Value addArray() { return addLeaf(new ArrayValue(names)); } - public final Value addObject() { return addLeaf(new ObjectValue(names)); } + public Value addArray() { return addLeaf(new ArrayValue(names)); } + public Value addObject() { return addLeaf(new ObjectValue(names)); } + } diff --git a/vespajlib/src/main/java/com/yahoo/slime/BufferedOutput.java b/vespajlib/src/main/java/com/yahoo/slime/BufferedOutput.java index 0fabea77df0..4c76abd4707 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/BufferedOutput.java +++ b/vespajlib/src/main/java/com/yahoo/slime/BufferedOutput.java @@ -3,6 +3,8 @@ package com.yahoo.slime; import com.yahoo.compress.Compressor; +import java.nio.charset.Charset; + final class BufferedOutput { private byte[] buf; @@ -56,6 +58,9 @@ final class BufferedOutput { System.arraycopy(buf, 0, ret, 0, pos); return ret; } + public String toString(Charset charset) { + return new String(buf, 0, pos, charset); + } Compressor.Compression compress(Compressor compressor) { return compressor.compress(buf, pos); } diff --git a/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java b/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java index b7f11f53cd5..682eccdab42 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java +++ b/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java @@ -2,10 +2,9 @@ package com.yahoo.slime; import com.yahoo.text.Text; -import com.yahoo.text.Utf8; -import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; /** * A port of the C++ json decoder intended to be fast. @@ -20,7 +19,7 @@ public class JsonDecoder { private final SlimeInserter slimeInserter = new SlimeInserter(null); private final ArrayInserter arrayInserter = new ArrayInserter(null); private final ObjectInserter objectInserter = new ObjectInserter(null, null); - private final ByteArrayOutputStream buf = new ByteArrayOutputStream(); + private final BufferedOutput buf = new BufferedOutput(); private static final byte[] TRUE = {'t', 'r', 'u', 'e'}; private static final byte[] FALSE = {'f', 'a', 'l', 's', 'e'}; @@ -86,15 +85,15 @@ public class JsonDecoder { case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': case '+': case '-': - buf.write(c); + buf.put(c); next(); break; default: if (likelyFloatingPoint) { - double num = Double.parseDouble(Utf8.toString(buf.toByteArray())); + double num = Double.parseDouble(buf.toString(StandardCharsets.UTF_8)); inserter.insertDOUBLE(num); } else { - long num = Long.parseLong(Utf8.toString(buf.toByteArray())); + long num = Long.parseLong(buf.toString(StandardCharsets.UTF_8)); inserter.insertLONG(num); } return; @@ -103,8 +102,8 @@ public class JsonDecoder { } private void expect(byte[] expected) { - for (int i = 0; i < expected.length; i++) { - if ( ! skip(expected[i])) { + for (byte b : expected) { + if ( ! skip(b)) { in.fail("Unexpected " + characterToReadableString(c)); return; } @@ -150,9 +149,9 @@ public class JsonDecoder { default: for (;;) { switch (c) { - case ':': case ' ': case '\t': case '\n': case '\r': case '\0': return Utf8.toString(buf.toByteArray()); + case ':': case ' ': case '\t': case '\n': case '\r': case '\0': return buf.toString(StandardCharsets.UTF_8); default: - buf.write(c); + buf.put(c); next(); break; } @@ -176,13 +175,13 @@ public class JsonDecoder { next(); switch (c) { case '"': case '\\': case '/': case '\'': - buf.write(c); + buf.put(c); break; - case 'b': buf.write((byte) '\b'); break; - case 'f': buf.write((byte) '\f'); break; - case 'n': buf.write((byte) '\n'); break; - case 'r': buf.write((byte) '\r'); break; - case 't': buf.write((byte) '\t'); break; + case 'b': buf.put((byte) '\b'); break; + case 'f': buf.put((byte) '\f'); break; + case 'n': buf.put((byte) '\n'); break; + case 'r': buf.put((byte) '\r'); break; + case 't': buf.put((byte) '\t'); break; case 'u': writeUtf8(dequoteUtf16(), buf, 0xffffff80); continue; default: in.fail("Invalid quoted char(" + c + ")"); @@ -193,34 +192,34 @@ public class JsonDecoder { case '"': case '\'': if (c == quote) { next(); - return Utf8.toString(buf.toByteArray()); + return buf.toString(StandardCharsets.UTF_8); } else { - buf.write(c); + buf.put(c); next(); } break; case '\0': in.fail("Unterminated string"); - return Utf8.toString(buf.toByteArray()); + return buf.toString(StandardCharsets.UTF_8); default: - buf.write(c); + buf.put(c); next(); break; } } } - private static void writeUtf8(long codepoint, ByteArrayOutputStream buf, long mask) { + private static void writeUtf8(long codepoint, BufferedOutput buf, long mask) { if ((codepoint & mask) == 0) { - buf.write((byte) ((mask << 1) | codepoint)); + buf.put((byte) ((mask << 1) | codepoint)); } else { writeUtf8(codepoint >> 6, buf, mask >> (2 - ((mask >> 6) & 0x1))); - buf.write((byte) (0x80 | (codepoint & 0x3f))); + buf.put((byte) (0x80 | (codepoint & 0x3f))); } } - private static byte[] unicodeStart = {'\\', 'u'}; + private final static byte[] unicodeStart = {'\\', 'u'}; private long dequoteUtf16() { next(); long codepoint = readHexValue(4); diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index ab475e25387..3d4536d9249 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -36,6 +36,7 @@ import java.util.function.Function; import java.util.stream.Collectors; import static com.yahoo.text.Ascii7BitMatcher.charsAndNumbers; +import static com.yahoo.tensor.functions.ScalarFunctions.Hamming; /** * A multidimensional array which can be used in computations. @@ -241,6 +242,7 @@ public interface Tensor { default Tensor notEqual(Tensor argument) { return join(argument, (a, b) -> ( a != b ? 1.0 : 0.0)); } default Tensor approxEqual(Tensor argument) { return join(argument, (a, b) -> ( approxEquals(a,b) ? 1.0 : 0.0)); } default Tensor bit(Tensor argument) { return join(argument, (a,b) -> ((int)b < 8 && (int)b >= 0 && ((int)a & (1 << (int)b)) != 0) ? 1.0 : 0.0); } + default Tensor hamming(Tensor argument) { return join(argument, (a,b) -> Hamming.hamming(a,b)); } default Tensor avg() { return avg(Collections.emptyList()); } default Tensor avg(String dimension) { return avg(Collections.singletonList(dimension)); } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunctions.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunctions.java index 3ee9e67cdd6..d6fcd17b8fb 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunctions.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunctions.java @@ -33,6 +33,7 @@ public class ScalarFunctions { public static DoubleBinaryOperator pow() { return new Pow(); } public static DoubleBinaryOperator squareddifference() { return new SquaredDifference(); } public static DoubleBinaryOperator subtract() { return new Subtract(); } + public static DoubleBinaryOperator hamming() { return new Hamming(); } public static DoubleUnaryOperator abs() { return new Abs(); } public static DoubleUnaryOperator acos() { return new Acos(); } @@ -152,6 +153,26 @@ public class ScalarFunctions { public String toString() { return "f(a,b)(a - b)"; } } + + public static class Hamming implements DoubleBinaryOperator { + public static double hamming(double left, double right) { + double distance = 0; + byte a = (byte) left; + byte b = (byte) right; + for (int i = 0; i < 8; i++) { + byte bit = (byte) (1 << i); + if ((a & bit) != (b & bit)) { + distance += 1; + } + } + return distance; + } + @Override + public double applyAsDouble(double left, double right) { return hamming(left, right); } + @Override + public String toString() { return "f(a,b)(hamming(a,b))"; } + } + // Unary operators ------------------------------------------------------------------------------ |